Merge pull request #1912 from pikasTech/fix/web-probe-memory-guard
Pipelines as Code CI / hwlab-web-probe-sentinel-nc01- Failed
Pipelines as Code CI / platform-infra-gitea-nc01- Success
Pipelines as Code CI / unidesk-host- Success

fix: 补齐 WebProbe 全入口浏览器启动门禁
This commit is contained in:
Lyon
2026-07-13 16:48:17 +08:00
committed by GitHub
36 changed files with 5104 additions and 1744 deletions
@@ -0,0 +1,123 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { readFileSync } from "node:fs";
import { test } from "bun:test";
import { rootPath } from "./config";
const webObserveSource = readFileSync(rootPath("scripts/src/gc-remote-web-observe.py"), "utf8");
const runnerSource = readFileSync(rootPath("scripts/src/gc-remote-runner.py"), "utf8");
function runPythonFixture(body: string): Record<string, unknown> {
const fixture = [
"import json, os, re, signal, time",
"PROVIDER_ID = 'NC01'",
"def safe_int(value, fallback=0):",
" try: return int(value)",
" except (TypeError, ValueError): return fallback",
"def config_list(config, key, fallback):",
" value = config.get(key, fallback)",
" return value if isinstance(value, list) else fallback",
"def config_int(config, key, fallback, minimum=None, maximum=None):",
" value = config.get(key, fallback)",
" return value if isinstance(value, int) and not isinstance(value, bool) else fallback",
"def config_float(config, key, fallback, minimum=None, maximum=None): return float(config.get(key, fallback))",
"def fmt_bytes(value): return str(value)",
"def redact_command_preview(value): return str(value)[:200]",
"def observe_run_record(path, stale_hours): return {}",
webObserveSource,
body,
].join("\n");
const result = spawnSync("python3", ["-"], { input: fixture, encoding: "utf8" });
assert.equal(result.status, 0, result.stderr || result.stdout);
return JSON.parse(result.stdout.trim()) as Record<string, unknown>;
}
test("memory-pressure candidate fixtures protect active observers and classify only closed stale/orphan trees", () => {
const result = runPythonFixture(String.raw`
MEMORY_CONFIG = {
"observeStateRoots": ["/tmp/observe"], "minimumReadableObserveRoots": 1,
"observeRunDepth": 6, "observeScanLimit": 100, "staleRunMaxAgeHours": 1,
"processScanLimit": 100, "processStaleMinAgeSeconds": 3600,
"maxMemoryReclaimCandidates": 20, "terminationGraceSeconds": 1,
"terminationPollMilliseconds": 50, "observerRootPatterns": ["observer-runner.mjs"],
"browserRootPatterns": ["observer-runner.mjs", "cliDaemon.mjs"],
"browserProcessPatterns": ["chromium", "/usr/bin/chromium"]
}
def row(pid, ppid, sid, start, age, command, rss=1024):
return {"pid": pid, "ppid": ppid, "sid": sid, "startTicks": start, "ageSeconds": age, "rssBytes": rss, "commandLine": command, "commandPreview": command}
processes = {
100: row(100, 1, 100, 1000, 7200, "node observer-runner.mjs"),
101: row(101, 100, 100, 1001, 7200, "/usr/bin/chromium --type=browser"),
200: row(200, 1, 200, 2000, 7200, "node observer-runner.mjs"),
300: row(300, 1, 300, 3000, 7200, "node cliDaemon.mjs"),
301: row(301, 300, 300, 3001, 7200, "/usr/bin/chromium --type=browser"),
}
read_proc_process_table = lambda: processes
observer_run_records = lambda: [
{"pid": 100, "pidAlive": True, "staleLiveSignal": False},
{"pid": 200, "pidAlive": True, "staleLiveSignal": True},
]
candidates = collect_memory_reclaim_candidates()
print(json.dumps({"kinds": sorted(item["kind"] for item in candidates), "roots": sorted(item["rootPid"] for item in candidates), "diagnostics": memory_reclaim_scan_diagnostics()}))
`);
assert.deepEqual(result.kinds, ["browser-orphan-process-tree", "stale-web-observer-process-tree"]);
assert.deepEqual(result.roots, [200, 300]);
assert.equal((result.diagnostics as Record<string, unknown>).blocked, false);
});
test("memory-pressure process scans fail closed on empty or truncated process tables", () => {
const result = runPythonFixture(String.raw`
MEMORY_CONFIG = {
"observeStateRoots": ["/tmp/observe"], "minimumReadableObserveRoots": 1,
"observeRunDepth": 6, "observeScanLimit": 100, "staleRunMaxAgeHours": 1,
"processScanLimit": 100, "processStaleMinAgeSeconds": 3600,
"maxMemoryReclaimCandidates": 20, "terminationGraceSeconds": 1,
"terminationPollMilliseconds": 50, "observerRootPatterns": ["observer-runner.mjs"],
"browserRootPatterns": ["cliDaemon.mjs"], "browserProcessPatterns": ["chromium"]
}
observer_run_records = lambda: []
read_proc_process_table = lambda: {}
empty = collect_memory_reclaim_candidates()
empty_diag = memory_reclaim_scan_diagnostics()
def truncated_scan():
global _PROCESS_SCAN_TRUNCATED
_PROCESS_SCAN_TRUNCATED = True
return {1: {"pid": 1, "ppid": 0, "sid": 1, "startTicks": 1, "ageSeconds": 7200, "rssBytes": 1, "commandLine": "node cliDaemon.mjs", "commandPreview": "node cliDaemon.mjs"}}
read_proc_process_table = truncated_scan
truncated = collect_memory_reclaim_candidates()
truncated_diag = memory_reclaim_scan_diagnostics()
print(json.dumps({"emptyCount": len(empty), "emptyBlocked": empty_diag["blocked"], "emptyError": empty_diag["processScanError"], "truncatedCount": len(truncated), "truncatedBlocked": truncated_diag["blocked"], "truncatedFlag": truncated_diag["processScanTruncated"]}))
`);
assert.equal(result.emptyCount, 0);
assert.equal(result.emptyBlocked, true);
assert.equal(result.emptyError, "proc-process-table-empty");
assert.equal(result.truncatedCount, 0);
assert.equal(result.truncatedBlocked, true);
assert.equal(result.truncatedFlag, true);
});
test("post-TERM identity matching tolerates reparenting but never PID startTicks reuse", () => {
const result = runPythonFixture(String.raw`
identity = {"pid": 41, "ppid": 10, "sid": 10, "startTicks": 1234}
reparented = {41: {"pid": 41, "ppid": 1, "sid": 41, "startTicks": 1234}}
reused = {41: {"pid": 41, "ppid": 10, "sid": 10, "startTicks": 9999}}
print(json.dumps({
"preTermReparented": matching_live_identity(identity, reparented) is not None,
"postTermReparented": matching_surviving_identity(identity, reparented) is not None,
"postTermReused": matching_surviving_identity(identity, reused) is not None,
}))
`);
assert.equal(result.preTermReparented, false);
assert.equal(result.postTermReparented, true);
assert.equal(result.postTermReused, false);
});
test("memory-pressure async runner uses a parent-child barrier and locked worker state", () => {
assert.match(runnerSource, /os\.pipe\(\)/u);
assert.match(runnerSource, /os\.read\(read_descriptor/u);
assert.match(runnerSource, /workerStartTicks/u);
assert.match(runnerSource, /write_memory_job_state\(/u);
assert.match(runnerSource, /workerAlive/u);
assert.match(runnerSource, /worker-identity-unavailable|worker-exited-without-terminal-state/u);
});
+418 -28
View File
@@ -1,8 +1,10 @@
import base64
import calendar
import fcntl
import json
import os
import re
import signal
import shutil
import subprocess
import sys
@@ -207,8 +209,41 @@ def job_paths(job_id):
return {
"state": os.path.join(REMOTE_GC_JOB_DIR, "%s.json" % job_id),
"log": os.path.join(REMOTE_GC_JOB_DIR, "%s.log" % job_id),
"lock": os.path.join(REMOTE_GC_JOB_DIR, "%s.lock" % job_id),
}
def process_identity(pid):
try:
with open("/proc/%s/stat" % int(pid), "r", encoding="utf-8") as handle:
text = handle.read().strip()
close = text.rfind(")")
fields = text[close + 2:].split() if close >= 0 else []
return {"state": fields[0], "startTicks": int(fields[19])} if len(fields) >= 20 else None
except (OSError, ValueError, IndexError):
return None
def process_start_ticks(pid):
identity = process_identity(pid)
return identity.get("startTicks") if identity else None
def acquire_job_lock(paths):
descriptor = os.open(paths["lock"], os.O_CREAT | os.O_RDWR, 0o600)
fcntl.flock(descriptor, fcntl.LOCK_EX)
return descriptor
def release_job_lock(descriptor):
try:
fcntl.flock(descriptor, fcntl.LOCK_UN)
finally:
os.close(descriptor)
def write_memory_job_state(paths, payload):
descriptor = acquire_job_lock(paths)
try:
write_json_atomic(paths["state"], payload)
finally:
release_job_lock(descriptor)
def status_command(job_id):
return "bun scripts/cli.ts gc remote %s status --job-id %s" % (PROVIDER_ID, job_id)
@@ -326,9 +361,31 @@ def remote_gc_job_status():
"statePath": paths["state"],
"logTail": read_file_tail(paths["log"]),
}
descriptor = None
try:
descriptor = acquire_job_lock(paths)
with open(paths["state"], "r", encoding="utf-8") as handle:
payload = json.load(handle)
if payload.get("scope") == "memory-pressure-only" and payload.get("status") == "running":
worker_pid = safe_int(payload.get("workerPid"))
expected_start_ticks = safe_int(payload.get("workerStartTicks"))
actual_identity = process_identity(worker_pid) if worker_pid > 0 else None
actual_start_ticks = actual_identity.get("startTicks") if actual_identity else None
worker_alive = actual_identity is not None and actual_identity.get("state") not in set(["Z", "X"]) \
and actual_start_ticks == expected_start_ticks
if not worker_alive:
payload.update({
"ok": False,
"status": "failed",
"outcome": "memory-reclaim-worker-exited-without-terminal-state",
"workerAlive": False,
"workerObservedState": actual_identity.get("state") if actual_identity else None,
"workerObservedStartTicks": actual_start_ticks,
"finishedAt": now_iso(),
})
write_json_atomic(paths["state"], payload)
else:
payload["workerAlive"] = True
except Exception as exc:
return {
"ok": False,
@@ -338,6 +395,9 @@ def remote_gc_job_status():
"statePath": paths["state"],
"logTail": read_file_tail(paths["log"]),
}
finally:
if descriptor is not None:
release_job_lock(descriptor)
payload["logTail"] = read_file_tail(paths["log"])
return payload
@@ -569,24 +629,50 @@ def collect_process_pressure(patterns):
}
def collect_memory_snapshot():
result = command(["free", "-b"], 5)
if result["exitCode"] != 0:
return {"ok": False, "error": "free-failed", "command": bounded(result)}
memory = {}
for line in result["stdout"].splitlines():
parts = line.split()
if parts and parts[0].rstrip(":") == "Mem" and len(parts) >= 7:
memory = {
"totalBytes": safe_int(parts[1]),
"usedBytes": safe_int(parts[2]),
"freeBytes": safe_int(parts[3]),
"availableBytes": safe_int(parts[6]),
"totalHuman": fmt_bytes(parts[1]),
"usedHuman": fmt_bytes(parts[2]),
"availableHuman": fmt_bytes(parts[6]),
}
break
return {"ok": bool(memory), "memory": memory, "command": bounded(result)}
source = "/proc/meminfo"
fields = {}
try:
with open(source, "r", encoding="utf-8") as handle:
for line in handle:
if ":" not in line:
continue
key, raw = line.split(":", 1)
parts = raw.strip().split()
if not parts:
continue
fields[key] = safe_int(parts[0]) * 1024
except OSError as exc:
return {
"ok": False,
"error": "proc-meminfo-read-failed",
"metric": "MemAvailable",
"source": source,
"swapExcluded": True,
"errorType": type(exc).__name__,
}
available_bytes = fields.get("MemAvailable")
if available_bytes is None:
return {
"ok": False,
"error": "memavailable-missing",
"metric": "MemAvailable",
"source": source,
"swapExcluded": True,
}
total_bytes = fields.get("MemTotal")
memory = {
"totalBytes": total_bytes,
"availableBytes": available_bytes,
"totalHuman": fmt_bytes(total_bytes) if total_bytes is not None else None,
"availableHuman": fmt_bytes(available_bytes),
}
return {
"ok": True,
"metric": "MemAvailable",
"source": source,
"swapExcluded": True,
"memory": memory,
}
def observe_run_record(path, stale_hours):
stat = os.stat(path)
@@ -608,6 +694,14 @@ def observe_run_record(path, stale_hours):
pass
if pid is not None:
break
live_freshness_timestamp = None
for source in [heartbeat, manifest]:
for key in ["updatedAt", "completedAt", "finishedAt", "stoppedAt"]:
live_freshness_timestamp = iso_or_epoch_to_epoch(source.get(key))
if live_freshness_timestamp is not None:
break
if live_freshness_timestamp is not None:
break
timestamp = None
for source in [heartbeat, manifest]:
for key in ["updatedAt", "completedAt", "finishedAt", "stoppedAt", "startedAt", "createdAt"]:
@@ -622,7 +716,8 @@ def observe_run_record(path, stale_hours):
status = heartbeat.get("status") or manifest.get("status") or manifest.get("state")
alive = pid_alive(pid)
terminal = str(status or "").lower() in set(["done", "completed", "complete", "failed", "blocked", "timeout", "timed-out", "stopped", "exited"])
stale_signal = (not alive) and age_hours >= float(stale_hours) and (terminal or status is None)
age_stale = age_hours >= float(stale_hours)
stale_signal = (not alive) and age_stale and (terminal or status is None)
return {
"id": os.path.basename(path),
"path": path,
@@ -632,6 +727,9 @@ def observe_run_record(path, stale_hours):
"ageHours": round(age_hours, 2),
"timestampBasis": "manifest-or-heartbeat" if heartbeat or manifest else "directory-mtime-fallback",
"staleSignal": stale_signal,
"staleLiveSignal": alive and age_stale and live_freshness_timestamp is not None,
"activeSignal": alive and not (age_stale and live_freshness_timestamp is not None),
"liveStaleEvidence": "heartbeat-or-manifest-update" if live_freshness_timestamp is not None else None,
"classification": "review-only",
}
@@ -647,6 +745,7 @@ def collect_web_observe_summary():
}
root_rows = []
stale_rows = []
stale_live_rows = []
active_rows = []
run_count = 0
total_bytes = 0
@@ -665,22 +764,18 @@ def collect_web_observe_summary():
"activeSignalCount": 0,
}
if exists:
try:
children = [os.path.join(root, name) for name in os.listdir(root)]
except OSError:
children = []
for child in children:
if not os.path.isdir(child):
continue
for child in configured_observe_runs(root):
try:
record = observe_run_record(child, stale_hours)
except OSError:
continue
row["runCount"] += 1
run_count += 1
if record.get("pidAlive"):
if record.get("activeSignal"):
row["activeSignalCount"] += 1
active_rows.append(record)
if record.get("staleLiveSignal"):
stale_live_rows.append(record)
if record.get("staleSignal"):
row["staleSignalCount"] += 1
stale_rows.append(record)
@@ -697,9 +792,11 @@ def collect_web_observe_summary():
"runCount": run_count,
"activeSignalCount": len(active_rows),
"staleSignalCount": len(stale_rows),
"staleLiveSignalCount": len(stale_live_rows),
"roots": root_rows,
"activeSignals": active_rows[:int(OPTIONS.get("limit") or 50)],
"staleSignals": stale_rows[:int(OPTIONS.get("limit") or 50)],
"staleLiveSignals": stale_live_rows[:int(OPTIONS.get("limit") or 50)],
"policy": "analysis-only; active or stale observe runs must be stopped/retained through controlled observer lifecycle commands, not raw process kill or directory deletion",
}
@@ -967,6 +1064,8 @@ def collect_protected():
return result
def collect_candidates(observed_at):
if bool(OPTIONS.get("memoryPressureOnly")):
return collect_memory_reclaim_candidates()
candidates = []
if OPTIONS.get("journal", True):
usage = command(["journalctl", "--disk-usage"], 5)
@@ -1314,6 +1413,29 @@ def summarize(candidates, returned, disk=None):
"target": target_assessment(disk, total),
}
def summarize_memory_candidates(candidates, returned):
total_memory = sum(safe_int(item.get("expectedMemoryRssBytes")) for item in candidates)
returned_memory = sum(safe_int(item.get("expectedMemoryRssBytes")) for item in returned)
by_kind = {}
for item in candidates:
kind = item.get("kind") or "unknown"
bucket = by_kind.setdefault(kind, {"count": 0, "expectedMemoryRssBytes": 0})
bucket["count"] += 1
bucket["expectedMemoryRssBytes"] += safe_int(item.get("expectedMemoryRssBytes"))
bucket["expectedMemoryRss"] = fmt_bytes(bucket["expectedMemoryRssBytes"])
return {
"scope": "memory-pressure-only",
"candidateCount": len(candidates),
"returnedCandidateCount": len(returned),
"expectedMemoryRssBytes": total_memory,
"expectedMemoryRss": fmt_bytes(total_memory),
"returnedExpectedMemoryRssBytes": returned_memory,
"returnedExpectedMemoryRss": fmt_bytes(returned_memory),
"estimatedDiskBytes": 0,
"outcome": "memory-reclaim-candidates" if candidates else "no-memory-reclaim",
"byKind": by_kind,
}
def assert_tmp_candidate(path):
resolved = os.path.abspath(path)
if not resolved.startswith("/tmp/"):
@@ -1400,6 +1522,8 @@ def execute(candidate):
before = du_size(path, 15) or path_size(path)
shutil.rmtree(path, ignore_errors=True)
return {"reclaimedBytes": before, "stale": record}
if kind in set(["browser-orphan-process-tree", "stale-web-observer-process-tree"]):
return execute_memory_process_candidate(candidate)
if kind == "k3s-cri-image-prune":
return execute_k3s_image_cache_prune()
if kind == "host-containerd-cache-prune":
@@ -1450,6 +1574,13 @@ def returned_results(results):
"path",
"status",
"estimatedReclaimBytes",
"estimatedDiskBytes",
"expectedMemoryRssBytes",
"reclaimedMemoryBytes",
"processCount",
"termSignalCount",
"killSignalCount",
"remainingIdentityCount",
"reclaimedBytes",
"orphanCount",
"selectedOrphanCount",
@@ -1468,6 +1599,38 @@ def returned_results(results):
return [compact_result(item) for item in (failed + started + succeeded)[:int(OPTIONS.get("resultLimit") or 50)]]
def plan_payload(observed_at, preflight, protected, candidates, visible):
if bool(OPTIONS.get("memoryPressureOnly")):
scan = memory_reclaim_scan_diagnostics()
summary = summarize_memory_candidates(candidates, visible)
if scan.get("blocked"):
summary["outcome"] = "memory-reclaim-scan-blocked"
return {
"ok": scan.get("ok") is True,
"action": "gc remote plan",
"providerId": PROVIDER_ID,
"scope": "memory-pressure-only",
"dryRun": True,
"mutation": False,
"observedAt": observed_at,
"options": OPTIONS,
"memoryBefore": collect_memory_snapshot(),
"summary": summary,
"candidateScan": scan,
"candidates": visible,
"protected": {
"activeObserverIntersection": "excluded" if scan.get("ok") is True else "unknown-protected-all",
"processIdentity": "pre-term:pid-ppid-sid-startTicks;post-term:pid-startTicks",
"treeClosure": "unique-root-ppid-sid-closed",
},
"policy": {
"requiresRunConfirm": True,
"planCommand": "bun scripts/cli.ts gc remote %s plan --memory-pressure-only" % PROVIDER_ID,
"runCommand": "bun scripts/cli.ts gc remote %s run --confirm --memory-pressure-only" % PROVIDER_ID,
"candidateKinds": ["browser-orphan-process-tree", "stale-web-observer-process-tree"],
"neverTouches": ["disk candidates", "apt cache", "journal", "tmp", "container images", "active observer process trees"],
"termination": "SIGTERM, bounded wait, identity reclose, SIGKILL only same-tree residual",
},
}
disk = df_snapshot()
ci_storage = ci_storage_snapshot()
memory_pressure = collect_memory_pressure()
@@ -1793,8 +1956,235 @@ def remote_policy_install_payload(observed_at):
},
}
def update_memory_pressure_job_progress(paths, completed_count, candidate_count, worker_pid, worker_start_ticks):
descriptor = acquire_job_lock(paths)
try:
with open(paths["state"], "r", encoding="utf-8") as handle:
payload = json.load(handle)
if payload.get("status") != "running" or safe_int(payload.get("workerPid")) != worker_pid \
or safe_int(payload.get("workerStartTicks")) != worker_start_ticks:
raise RuntimeError("memory pressure job state no longer belongs to this worker")
payload.update({
"workerHeartbeatAt": now_iso(),
"progress": {"completedCount": completed_count, "candidateCount": candidate_count},
})
write_json_atomic(paths["state"], payload)
finally:
release_job_lock(descriptor)
def finish_memory_pressure_job(job_id, candidates, observed_at, worker_pid, worker_start_ticks):
paths = job_paths(job_id)
memory_before = collect_memory_snapshot()
results = []
update_memory_pressure_job_progress(paths, 0, len(candidates), worker_pid, worker_start_ticks)
for index, candidate in enumerate(candidates):
try:
execution = execute(candidate)
item = dict(candidate)
item.update({
"status": execution.get("status") or "succeeded",
"reclaimedBytes": 0,
"reclaimedMemoryBytes": execution.get("reclaimedMemoryBytes"),
"termSignalCount": execution.get("termSignalCount"),
"killSignalCount": execution.get("killSignalCount"),
"remainingIdentityCount": execution.get("remainingIdentityCount"),
"remainingIdentities": execution.get("remainingIdentities"),
"lateAllowlistedIdentityCount": execution.get("lateAllowlistedIdentityCount"),
"memoryBefore": execution.get("memoryBefore"),
"memoryAfter": execution.get("memoryAfter"),
})
results.append(item)
except Exception as exc:
item = dict(candidate)
item.update({"status": "failed", "reclaimedBytes": 0, "reclaimedMemoryBytes": None, "error": str(exc)})
results.append(item)
update_memory_pressure_job_progress(paths, index + 1, len(candidates), worker_pid, worker_start_ticks)
memory_after = collect_memory_snapshot()
failed = [item for item in results if item.get("status") == "failed"]
returned = returned_results(results)
summary = summarize_memory_candidates(candidates, returned)
summary.update({
"attemptedCount": len(results),
"succeededCount": len([item for item in results if item.get("status") == "succeeded"]),
"failedCount": len(failed),
"actualMemoryAvailableDeltaBytes": (
safe_int((memory_after.get("memory") or {}).get("availableBytes"))
- safe_int((memory_before.get("memory") or {}).get("availableBytes"))
) if memory_before.get("ok") and memory_after.get("ok") else None,
})
payload = {
"ok": len(failed) == 0,
"action": "gc remote run",
"providerId": PROVIDER_ID,
"scope": "memory-pressure-only",
"status": "succeeded" if len(failed) == 0 else "failed",
"jobId": job_id,
"statusCommand": status_command(job_id),
"statePath": paths["state"],
"dryRun": False,
"mutation": len(results) > 0,
"observedAt": observed_at,
"startedAt": observed_at,
"finishedAt": now_iso(),
"workerPid": worker_pid,
"workerStartTicks": worker_start_ticks,
"workerAlive": False,
"workerCompleted": True,
"workerHeartbeatAt": now_iso(),
"progress": {"completedCount": len(candidates), "candidateCount": len(candidates)},
"memoryBefore": memory_before,
"memoryAfter": memory_after,
"summary": summary,
"results": returned,
}
write_memory_job_state(paths, payload)
def start_memory_pressure_job(observed_at, candidates):
job_id = "%s-memory-pressure-%s-%s" % (
re.sub(r"[^A-Za-z0-9._-]+", "-", PROVIDER_ID.lower()).strip("-") or "provider",
int(time.time()),
os.getpid(),
)
paths = job_paths(job_id)
summary = summarize_memory_candidates(candidates, candidates)
scan = memory_reclaim_scan_diagnostics()
base = {
"ok": scan.get("ok") is True,
"action": "gc remote run",
"providerId": PROVIDER_ID,
"scope": "memory-pressure-only",
"jobId": job_id,
"statusCommand": status_command(job_id),
"statePath": paths["state"],
"dryRun": False,
"mutation": len(candidates) > 0,
"observedAt": observed_at,
"startedAt": observed_at,
"summary": summary,
"candidateScan": scan,
}
if scan.get("blocked"):
terminal = dict(base)
terminal.update({
"status": "blocked",
"outcome": "memory-reclaim-scan-blocked",
"mutation": False,
"finishedAt": now_iso(),
"results": [],
})
write_json_atomic(paths["state"], terminal)
return terminal
if not candidates:
terminal = dict(base)
terminal.update({
"status": "blocked",
"outcome": "no-memory-reclaim-candidates",
"mutation": False,
"finishedAt": now_iso(),
"results": [],
})
write_json_atomic(paths["state"], terminal)
return terminal
read_descriptor, write_descriptor = os.pipe()
try:
child_pid = os.fork()
except OSError as exc:
os.close(read_descriptor)
os.close(write_descriptor)
failed = dict(base)
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-fork-failed", "error": str(exc), "finishedAt": now_iso()})
write_memory_job_state(paths, failed)
return failed
if child_pid == 0:
try:
os.close(write_descriptor)
released = os.read(read_descriptor, 1)
os.close(read_descriptor)
if released != b"1":
os._exit(1)
os.setsid()
with open("/dev/null", "rb", buffering=0) as stdin_handle, open(paths["log"], "ab", buffering=0) as log_handle:
os.dup2(stdin_handle.fileno(), 0)
os.dup2(log_handle.fileno(), 1)
os.dup2(log_handle.fileno(), 2)
finish_memory_pressure_job(job_id, candidates, observed_at, os.getpid(), process_start_ticks(os.getpid()))
except Exception as exc:
failed = dict(base)
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-worker-failed", "workerPid": os.getpid(), "workerStartTicks": process_start_ticks(os.getpid()), "error": str(exc), "finishedAt": now_iso()})
try:
write_memory_job_state(paths, failed)
except Exception:
pass
finally:
os._exit(0)
os.close(read_descriptor)
worker_start_ticks = process_start_ticks(child_pid)
if worker_start_ticks is None:
os.close(write_descriptor)
try:
os.kill(child_pid, signal.SIGKILL)
except ProcessLookupError:
pass
failed = dict(base)
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-worker-identity-unavailable", "workerPid": child_pid, "finishedAt": now_iso()})
write_memory_job_state(paths, failed)
return failed
running = dict(base)
running.update({
"status": "running",
"outcome": "memory-reclaim-started",
"workerPid": child_pid,
"workerStartTicks": worker_start_ticks,
"workerAlive": True,
"workerHeartbeatAt": now_iso(),
"progress": {"completedCount": 0, "candidateCount": len(candidates)},
})
try:
write_memory_job_state(paths, running)
os.write(write_descriptor, b"1")
except Exception as exc:
try:
os.kill(child_pid, signal.SIGKILL)
except ProcessLookupError:
pass
failed = dict(base)
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-worker-release-failed", "workerPid": child_pid, "workerStartTicks": worker_start_ticks, "error": str(exc), "finishedAt": now_iso()})
write_memory_job_state(paths, failed)
return failed
finally:
os.close(write_descriptor)
return running
def main():
observed_at = now_iso()
if ACTION == "memory":
memory = collect_memory_snapshot()
emit_json({
"ok": memory.get("ok") is True,
"action": "gc remote memory",
"providerId": PROVIDER_ID,
"dryRun": True,
"mutation": False,
"observedAt": observed_at,
"metric": memory.get("metric"),
"source": memory.get("source"),
"swapExcluded": memory.get("swapExcluded") is True,
"memory": memory.get("memory"),
"error": memory.get("error"),
"valuesRedacted": True,
}, persist_large=False)
return 0
if ACTION == "status" and job_id_or_none():
emit_json(remote_gc_job_status(), persist_large=False)
return 0
if bool(OPTIONS.get("memoryPressureOnly")) and ACTION in set(["plan", "run"]):
candidates = collect_memory_reclaim_candidates()
visible = visible_items(candidates)
if ACTION == "plan":
emit_json(plan_payload(observed_at, {"skipped": True, "reason": "memory-pressure-only"}, [], candidates, visible), persist_large=True)
else:
emit_json(start_memory_pressure_job(observed_at, visible), persist_large=True)
return 0
preflight = cluster_preflight()
if ACTION == "policy-plan":
emit_json(remote_policy_plan_payload(observed_at), persist_large=False)
@@ -1854,7 +2244,7 @@ def main():
})
emit_json(snapshot, persist_large=True)
return 0
protected = collect_protected()
protected = [] if bool(OPTIONS.get("memoryPressureOnly")) else collect_protected()
candidates = collect_candidates(observed_at)
visible = visible_items(candidates)
if ACTION == "plan":
+473 -4
View File
@@ -1,14 +1,63 @@
_OBSERVE_RUN_CACHE = {}
_OBSERVE_SCAN_TRUNCATED = False
_PROCESS_SCAN_TRUNCATED = False
_OBSERVE_SCAN_ERROR = None
_PROCESS_SCAN_ERROR = None
_MEMORY_RECLAIM_CONFIG_ERROR = None
_OBSERVE_ROOT_DIAGNOSTICS = {"configured": [], "readable": [], "missing": []}
def configured_observe_roots():
roots = config_list(MEMORY_CONFIG, "observeStateRoots", config_list(MEMORY_CONFIG, "webObserveRoots", []))
return [os.path.abspath(item) for item in roots if isinstance(item, str) and item.startswith("/")]
def is_direct_observe_run_path(path):
def configured_observe_runs(root):
global _OBSERVE_SCAN_TRUNCATED, _OBSERVE_SCAN_ERROR
resolved_root = os.path.abspath(root)
if resolved_root in _OBSERVE_RUN_CACHE:
return list(_OBSERVE_RUN_CACHE[resolved_root])
max_depth = config_int(MEMORY_CONFIG, "observeRunDepth", 1, minimum=1, maximum=12)
scan_limit = config_int(MEMORY_CONFIG, "observeScanLimit", 1000, minimum=1, maximum=100000)
markers = set(["manifest.json", "heartbeat.json", "pid", "observer.pid", "runner.pid"])
runs = []
scanned = 0
if not os.path.isdir(resolved_root) or os.path.islink(resolved_root):
return runs
def record_walk_error(error):
global _OBSERVE_SCAN_ERROR
_OBSERVE_SCAN_ERROR = "observe-state-walk-failed:%s" % type(error).__name__
for current, directories, files in os.walk(resolved_root, topdown=True, followlinks=False, onerror=record_walk_error):
relative = os.path.relpath(current, resolved_root)
depth = 0 if relative == "." else relative.count(os.sep) + 1
directories[:] = sorted(name for name in directories if not os.path.islink(os.path.join(current, name)))
if depth >= max_depth:
directories[:] = []
if depth == 0:
continue
scanned += 1
if scanned > scan_limit:
_OBSERVE_SCAN_TRUNCATED = True
break
if markers.intersection(files):
runs.append(os.path.abspath(current))
directories[:] = []
if not _OBSERVE_SCAN_TRUNCATED:
_OBSERVE_RUN_CACHE[resolved_root] = list(runs)
return runs
def is_observe_run_path(path):
resolved = os.path.abspath(path)
for root in configured_observe_roots():
if os.path.dirname(resolved) == root and resolved.startswith(root.rstrip("/") + "/"):
prefix = root.rstrip("/") + "/"
if resolved.startswith(prefix) and resolved in set(configured_observe_runs(root)):
return True
return False
def is_direct_observe_run_path(path):
# Kept as a compatibility name for generic artifact retention. Validation is
# now marker/depth based because the configured roots contain dated trees.
return is_observe_run_path(path)
def path_has_open_fd(path):
resolved = os.path.realpath(path)
prefix = resolved.rstrip("/") + "/"
@@ -42,8 +91,8 @@ def path_has_open_fd(path):
def assert_web_observe_candidate(path):
resolved = os.path.abspath(path)
if not is_direct_observe_run_path(resolved):
raise RuntimeError("refusing to remove web-observe path outside configured direct run roots: %s" % path)
if not is_observe_run_path(resolved):
raise RuntimeError("refusing to remove web-observe path outside configured marker/depth run roots: %s" % path)
if os.path.islink(resolved) or not os.path.isdir(resolved):
raise RuntimeError("refusing to remove non-directory or symlink web-observe path: %s" % path)
stale_hours = config_float(MEMORY_CONFIG, "staleRunMaxAgeHours", 6.0, minimum=0.0)
@@ -55,3 +104,423 @@ def assert_web_observe_candidate(path):
if path_has_open_fd(resolved):
raise RuntimeError("refusing to remove web-observe run with open fd/cwd reference: %s" % path)
return record
def read_proc_process_table():
global _PROCESS_SCAN_TRUNCATED, _PROCESS_SCAN_ERROR
scan_limit = config_int(MEMORY_CONFIG, "processScanLimit", 16384, minimum=1, maximum=1000000)
try:
clock_ticks = int(os.sysconf("SC_CLK_TCK"))
uptime = float(open("/proc/uptime", "r", encoding="utf-8").read().split()[0])
page_size = int(os.sysconf("SC_PAGE_SIZE"))
pid_names = sorted((name for name in os.listdir("/proc") if name.isdigit()), key=int)
if len(pid_names) > scan_limit:
_PROCESS_SCAN_TRUNCATED = True
return {}
except Exception as exc:
_PROCESS_SCAN_ERROR = "proc-process-table-read-failed:%s" % type(exc).__name__
return {}
rows = {}
stat_read_failures = 0
for pid_name in pid_names:
base = os.path.join("/proc", pid_name)
try:
with open(os.path.join(base, "stat"), "r", encoding="utf-8") as handle:
stat_text = handle.read().strip()
close = stat_text.rfind(")")
if close < 0:
continue
fields = stat_text[close + 2:].split()
if len(fields) < 22:
continue
pid = int(pid_name)
ppid = int(fields[1])
sid = int(fields[3])
start_ticks = int(fields[19])
try:
with open(os.path.join(base, "cmdline"), "rb") as handle:
command_line = handle.read(65536).replace(b"\0", b" ").decode("utf-8", errors="replace").strip()
except OSError:
command_line = ""
try:
with open(os.path.join(base, "statm"), "r", encoding="utf-8") as handle:
rss_bytes = safe_int(handle.read().split()[1]) * page_size
except Exception:
rss_bytes = 0
rows[pid] = {
"pid": pid,
"ppid": ppid,
"sid": sid,
"startTicks": start_ticks,
"ageSeconds": max(0, int(uptime - (float(start_ticks) / float(clock_ticks)))),
"rssBytes": rss_bytes,
"commandPreview": redact_command_preview(command_line),
"commandLine": command_line,
}
except (OSError, ValueError, IndexError):
stat_read_failures += 1
continue
if not rows:
_PROCESS_SCAN_ERROR = "proc-process-table-empty"
elif stat_read_failures > max(32, len(pid_names) // 2):
_PROCESS_SCAN_ERROR = "proc-process-stat-read-failure-threshold:%s-of-%s" % (stat_read_failures, len(pid_names))
return rows
def validate_observe_run_markers(path):
for name in ["heartbeat.json", "manifest.json"]:
marker = os.path.join(path, name)
if not os.path.lexists(marker):
continue
if os.path.islink(marker) or not os.path.isfile(marker):
raise OSError("observe marker is not a regular file: %s" % marker)
with open(marker, "r", encoding="utf-8") as handle:
value = json.load(handle)
if not isinstance(value, dict):
raise OSError("observe JSON marker is not an object: %s" % marker)
for name in ["pid", "observer.pid", "browser.pid", "runner.pid"]:
marker = os.path.join(path, name)
if not os.path.lexists(marker):
continue
if os.path.islink(marker) or not os.path.isfile(marker):
raise OSError("observe pid marker is not a regular file: %s" % marker)
with open(marker, "r", encoding="utf-8") as handle:
raw = handle.read().strip()
if not re.match(r"^[1-9]\d*$", raw):
raise OSError("observe pid marker is invalid: %s" % marker)
def observer_run_records():
global _OBSERVE_SCAN_ERROR, _OBSERVE_ROOT_DIAGNOSTICS
stale_hours = config_float(MEMORY_CONFIG, "staleRunMaxAgeHours", 6.0, minimum=0.0)
records = []
roots = configured_observe_roots()
accessible_roots = []
missing_roots = []
for root in roots:
try:
root_stat = os.stat(root)
if os.path.isdir(root) and not os.path.islink(root):
accessible_roots.append(root)
elif root_stat:
_OBSERVE_SCAN_ERROR = "observe-state-root-not-directory"
except FileNotFoundError:
missing_roots.append(root)
continue
except OSError as exc:
_OBSERVE_SCAN_ERROR = "observe-state-root-read-failed:%s" % type(exc).__name__
minimum_readable = config_int(MEMORY_CONFIG, "minimumReadableObserveRoots", 1, minimum=1, maximum=100)
_OBSERVE_ROOT_DIAGNOSTICS = {"configured": list(roots), "readable": list(accessible_roots), "missing": missing_roots}
if len(accessible_roots) < minimum_readable:
_OBSERVE_SCAN_ERROR = _OBSERVE_SCAN_ERROR or "minimum-readable-observe-roots-not-met:%s-of-%s" % (len(accessible_roots), minimum_readable)
for root in accessible_roots:
for path in configured_observe_runs(root):
try:
validate_observe_run_markers(path)
records.append(observe_run_record(path, stale_hours))
except (OSError, ValueError, json.JSONDecodeError) as exc:
_OBSERVE_SCAN_ERROR = "observe-run-marker-read-failed:%s" % type(exc).__name__
continue
return records
def process_descendants(processes, root_pid):
selected = set([int(root_pid)])
changed = True
while changed:
changed = False
for pid, row in processes.items():
if pid not in selected and safe_int(row.get("ppid")) in selected:
selected.add(pid)
changed = True
return selected
def session_closed_tree(processes, sid):
members = {pid: row for pid, row in processes.items() if safe_int(row.get("sid")) == int(sid)}
if not members:
return None
roots = [pid for pid, row in members.items() if safe_int(row.get("ppid")) not in members]
if len(roots) != 1:
return None
root_pid = roots[0]
if process_descendants(members, root_pid) != set(members.keys()):
return None
return {"rootPid": root_pid, "members": members}
def command_matches(row, patterns):
haystack = str(row.get("commandLine") or "").lower()
return any(str(pattern).lower() in haystack for pattern in patterns if str(pattern))
def memory_process_candidate_id(kind, root):
return "%s:%s:%s:%s" % (kind, safe_int(root.get("sid")), safe_int(root.get("pid")), safe_int(root.get("startTicks")))
def collect_memory_reclaim_candidates():
global _OBSERVE_SCAN_TRUNCATED, _PROCESS_SCAN_TRUNCATED, _OBSERVE_SCAN_ERROR, _PROCESS_SCAN_ERROR, _MEMORY_RECLAIM_CONFIG_ERROR, _OBSERVE_ROOT_DIAGNOSTICS
_OBSERVE_SCAN_TRUNCATED = False
_PROCESS_SCAN_TRUNCATED = False
_OBSERVE_SCAN_ERROR = None
_PROCESS_SCAN_ERROR = None
_MEMORY_RECLAIM_CONFIG_ERROR = None
_OBSERVE_ROOT_DIAGNOSTICS = {"configured": [], "readable": [], "missing": []}
_OBSERVE_RUN_CACHE.clear()
required_keys = [
"observeStateRoots", "minimumReadableObserveRoots", "observeRunDepth", "observeScanLimit", "staleRunMaxAgeHours",
"processScanLimit", "processStaleMinAgeSeconds", "maxMemoryReclaimCandidates",
"terminationGraceSeconds", "terminationPollMilliseconds", "observerRootPatterns",
"browserRootPatterns", "browserProcessPatterns",
]
if any(key not in MEMORY_CONFIG for key in required_keys):
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-policy-missing"
return []
list_keys = ["observeStateRoots", "observerRootPatterns", "browserRootPatterns", "browserProcessPatterns"]
integer_keys = [
"observeRunDepth", "observeScanLimit", "processScanLimit", "processStaleMinAgeSeconds",
"maxMemoryReclaimCandidates", "terminationGraceSeconds", "terminationPollMilliseconds",
]
if any(not isinstance(MEMORY_CONFIG.get(key), list) or not MEMORY_CONFIG.get(key) for key in list_keys):
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-list-invalid"
return []
if any(any(not isinstance(item, str) or not item.strip() or "\0" in item for item in MEMORY_CONFIG.get(key)) for key in list_keys):
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-list-entry-invalid"
return []
if any(not os.path.isabs(item) for item in MEMORY_CONFIG.get("observeStateRoots")):
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-observe-root-not-absolute"
return []
integer_ranges = {
"observeRunDepth": (1, 12),
"minimumReadableObserveRoots": (1, 100),
"observeScanLimit": (1, 100000),
"processScanLimit": (1, 1000000),
"processStaleMinAgeSeconds": (60, 604800),
"maxMemoryReclaimCandidates": (1, 100),
"terminationGraceSeconds": (1, 30),
"terminationPollMilliseconds": (50, 1000),
}
if any(isinstance(MEMORY_CONFIG.get(key), bool) or not isinstance(MEMORY_CONFIG.get(key), int)
or MEMORY_CONFIG.get(key) < minimum or MEMORY_CONFIG.get(key) > maximum
for key, (minimum, maximum) in integer_ranges.items()):
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-integer-invalid"
return []
if MEMORY_CONFIG.get("minimumReadableObserveRoots") > len(MEMORY_CONFIG.get("observeStateRoots")):
_MEMORY_RECLAIM_CONFIG_ERROR = "minimum-readable-observe-roots-exceeds-configured-roots"
return []
stale_hours = MEMORY_CONFIG.get("staleRunMaxAgeHours")
if isinstance(stale_hours, bool) or not isinstance(stale_hours, (int, float)) or stale_hours <= 0 or stale_hours > 8760:
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-stale-age-invalid"
return []
browser_patterns = config_list(MEMORY_CONFIG, "browserProcessPatterns", [])
root_patterns = config_list(MEMORY_CONFIG, "browserRootPatterns", [])
observer_patterns = config_list(MEMORY_CONFIG, "observerRootPatterns", [])
stale_seconds = config_int(MEMORY_CONFIG, "processStaleMinAgeSeconds", 3600, minimum=60, maximum=604800)
if not browser_patterns or not root_patterns or not observer_patterns:
return []
processes = read_proc_process_table()
records = observer_run_records()
if not processes and _PROCESS_SCAN_ERROR is None:
_PROCESS_SCAN_ERROR = "proc-process-table-empty"
if _PROCESS_SCAN_TRUNCATED or _OBSERVE_SCAN_TRUNCATED or _PROCESS_SCAN_ERROR or _OBSERVE_SCAN_ERROR:
return []
fresh_observer_pids = set()
stale_observer_pids = set()
for record in records:
pid = safe_int(record.get("pid"))
if pid <= 0 or pid not in processes:
continue
if record.get("staleLiveSignal"):
stale_observer_pids.add(pid)
else:
fresh_observer_pids.add(pid)
protected_pids = set()
for pid in fresh_observer_pids:
protected_pids.update(process_descendants(processes, pid))
candidates = []
for sid in sorted(set(safe_int(row.get("sid")) for row in processes.values() if safe_int(row.get("sid")) > 0)):
closed = session_closed_tree(processes, sid)
if closed is None:
continue
members = closed["members"]
member_pids = set(members.keys())
if member_pids.intersection(protected_pids):
continue
root = members[closed["rootPid"]]
if safe_int(root.get("ageSeconds")) < stale_seconds:
continue
has_browser = any(command_matches(row, browser_patterns) for row in members.values())
intersecting_stale = member_pids.intersection(stale_observer_pids)
kind = None
reason = None
if intersecting_stale and command_matches(root, observer_patterns):
kind = "stale-web-observer-process-tree"
reason = "stale-observer-heartbeat"
elif not member_pids.intersection(set(safe_int(item.get("pid")) for item in records if item.get("pidAlive"))):
if command_matches(root, root_patterns) and has_browser:
kind = "browser-orphan-process-tree"
reason = "browser-session-without-observer-record"
if kind is None:
continue
identities = [
{
"pid": pid,
"ppid": safe_int(row.get("ppid")),
"sid": safe_int(row.get("sid")),
"startTicks": safe_int(row.get("startTicks")),
"rssBytes": safe_int(row.get("rssBytes")),
"commandPreview": row.get("commandPreview"),
}
for pid, row in sorted(members.items())
]
expected_memory = sum(safe_int(item.get("rssBytes")) for item in identities)
candidates.append({
"id": memory_process_candidate_id(kind, root),
"kind": kind,
"risk": "medium",
"description": "Terminate a YAML-allowlisted stale WebProbe process session after identity and active-observer revalidation",
"reason": reason,
"rootPid": safe_int(root.get("pid")),
"sid": sid,
"rootStartTicks": safe_int(root.get("startTicks")),
"processCount": len(identities),
"processIdentities": identities,
"expectedMemoryRssBytes": expected_memory,
"expectedMemoryRss": fmt_bytes(expected_memory),
"estimatedDiskBytes": 0,
"estimatedReclaimBytes": 0,
"activeObserverIntersection": False,
"treeClosure": "unique-root-ppid-sid-closed",
"action": {"op": "term-wait-kill-exact-process-tree", "allowlist": "yaml-web-probe-memory-pressure"},
})
max_candidates = config_int(MEMORY_CONFIG, "maxMemoryReclaimCandidates", 20, minimum=1, maximum=100)
return sorted(candidates, key=lambda item: safe_int(item.get("expectedMemoryRssBytes")), reverse=True)[:max_candidates]
def memory_reclaim_scan_diagnostics():
blocked = bool(_MEMORY_RECLAIM_CONFIG_ERROR or _OBSERVE_SCAN_TRUNCATED or _PROCESS_SCAN_TRUNCATED or _OBSERVE_SCAN_ERROR or _PROCESS_SCAN_ERROR)
return {
"ok": not blocked,
"blocked": blocked,
"configError": _MEMORY_RECLAIM_CONFIG_ERROR,
"observeScanTruncated": bool(_OBSERVE_SCAN_TRUNCATED),
"processScanTruncated": bool(_PROCESS_SCAN_TRUNCATED),
"observeScanError": _OBSERVE_SCAN_ERROR,
"processScanError": _PROCESS_SCAN_ERROR,
"observeRoots": {
"configuredCount": len(_OBSERVE_ROOT_DIAGNOSTICS.get("configured") or []),
"readableCount": len(_OBSERVE_ROOT_DIAGNOSTICS.get("readable") or []),
"missingCount": len(_OBSERVE_ROOT_DIAGNOSTICS.get("missing") or []),
"minimumReadable": safe_int(MEMORY_CONFIG.get("minimumReadableObserveRoots")),
"configured": _OBSERVE_ROOT_DIAGNOSTICS.get("configured") or [],
"readable": _OBSERVE_ROOT_DIAGNOSTICS.get("readable") or [],
"missing": _OBSERVE_ROOT_DIAGNOSTICS.get("missing") or [],
},
"policySource": "config/unidesk-cli.yaml#gc.remote.targets.%s.memoryPressure" % PROVIDER_ID,
}
def matching_live_identity(identity, processes):
pid = safe_int(identity.get("pid"))
row = processes.get(pid)
return row if row \
and safe_int(row.get("startTicks")) == safe_int(identity.get("startTicks")) \
and safe_int(row.get("ppid")) == safe_int(identity.get("ppid")) \
and safe_int(row.get("sid")) == safe_int(identity.get("sid")) else None
def matching_surviving_identity(identity, processes):
pid = safe_int(identity.get("pid"))
row = processes.get(pid)
return row if row \
and safe_int(row.get("startTicks")) == safe_int(identity.get("startTicks")) else None
def require_complete_process_scan(processes, phase):
if not processes or _PROCESS_SCAN_ERROR or _PROCESS_SCAN_TRUNCATED:
raise RuntimeError("memory process identity scan blocked during %s: %s" % (
phase,
_PROCESS_SCAN_ERROR or "process-scan-truncated",
))
return processes
def extend_with_allowlisted_session_members(identities, processes, sid, browser_patterns):
known = {(safe_int(item.get("pid")), safe_int(item.get("startTicks"))) for item in identities}
added = []
for pid, row in processes.items():
identity_key = (safe_int(pid), safe_int(row.get("startTicks")))
if identity_key in known or safe_int(row.get("sid")) != safe_int(sid) or not command_matches(row, browser_patterns):
continue
item = {
"pid": safe_int(pid),
"ppid": safe_int(row.get("ppid")),
"sid": safe_int(row.get("sid")),
"startTicks": safe_int(row.get("startTicks")),
"rssBytes": safe_int(row.get("rssBytes")),
"commandPreview": row.get("commandPreview"),
"discoveredAfterTerm": True,
}
identities.append(item)
added.append(item)
known.add(identity_key)
return added
def execute_memory_process_candidate(candidate):
current = next((item for item in collect_memory_reclaim_candidates() if item.get("id") == candidate.get("id")), None)
if current is None:
raise RuntimeError("memory candidate no longer has the same closed identity or is now protected")
before = collect_memory_snapshot()
identities = [dict(item) for item in (current.get("processIdentities") or [])]
candidate_sid = safe_int(current.get("sid"))
browser_patterns = config_list(MEMORY_CONFIG, "browserProcessPatterns", [])
late_members = []
processes = require_complete_process_scan(read_proc_process_table(), "pre-term-revalidation")
live = [item for item in identities if matching_live_identity(item, processes)]
for item in sorted(live, key=lambda value: safe_int(value.get("pid")), reverse=True):
try:
os.kill(safe_int(item.get("pid")), signal.SIGTERM)
except ProcessLookupError:
pass
grace_seconds = config_int(MEMORY_CONFIG, "terminationGraceSeconds", 5, minimum=1, maximum=30)
poll_ms = config_int(MEMORY_CONFIG, "terminationPollMilliseconds", 200, minimum=50, maximum=1000)
deadline = time.time() + grace_seconds
residual = []
while time.time() < deadline:
processes = require_complete_process_scan(read_proc_process_table(), "term-grace-poll")
late_members.extend(extend_with_allowlisted_session_members(identities, processes, candidate_sid, browser_patterns))
residual = [item for item in identities if matching_surviving_identity(item, processes)]
if not residual:
break
time.sleep(float(poll_ms) / 1000.0)
processes = require_complete_process_scan(read_proc_process_table(), "pre-kill-revalidation")
late_members.extend(extend_with_allowlisted_session_members(identities, processes, candidate_sid, browser_patterns))
residual = [item for item in identities if matching_surviving_identity(item, processes)]
for item in sorted(residual, key=lambda value: safe_int(value.get("pid")), reverse=True):
try:
os.kill(safe_int(item.get("pid")), signal.SIGKILL)
except ProcessLookupError:
pass
final_deadline = time.time() + min(2.0, float(grace_seconds))
remaining = residual
while time.time() < final_deadline:
processes = require_complete_process_scan(read_proc_process_table(), "kill-grace-poll")
late_members.extend(extend_with_allowlisted_session_members(identities, processes, candidate_sid, browser_patterns))
remaining = [item for item in identities if matching_surviving_identity(item, processes)]
if not remaining:
break
time.sleep(float(poll_ms) / 1000.0)
after = collect_memory_snapshot()
before_available = safe_int((before.get("memory") or {}).get("availableBytes"), None)
after_available = safe_int((after.get("memory") or {}).get("availableBytes"), None)
return {
"status": "failed" if remaining else "succeeded",
"reclaimedBytes": 0,
"reclaimedMemoryBytes": max(0, after_available - before_available) if before_available is not None and after_available is not None else None,
"expectedMemoryRssBytes": current.get("expectedMemoryRssBytes"),
"termSignalCount": len(live),
"killSignalCount": len(residual),
"remainingIdentityCount": len(remaining),
"lateAllowlistedIdentityCount": len(late_members),
"remainingIdentities": [
{
"pid": safe_int(item.get("pid")),
"startTicks": safe_int(item.get("startTicks")),
"sid": safe_int(item.get("sid")),
"originalPpid": safe_int(item.get("ppid")),
"observedPpid": safe_int((matching_surviving_identity(item, processes) or {}).get("ppid")),
}
for item in remaining
],
"memoryBefore": before,
"memoryAfter": after,
"identityRevalidated": True,
"activeObserverIntersection": False,
}
+20 -5
View File
@@ -5,7 +5,7 @@ import { type UniDeskConfig, rootPath } from "./config";
import { remoteGcDegradedFailure } from "./gc-remote-degraded";
import { runSshCommandCapture } from "./ssh";
type RemoteGcAction = "plan" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status";
type RemoteGcAction = "plan" | "memory" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status";
interface RemoteGcOptions {
confirm: boolean;
@@ -36,6 +36,7 @@ interface RemoteGcOptions {
full: boolean;
historyLimit: number;
saveSnapshot: boolean;
memoryPressureOnly: boolean;
}
const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
@@ -65,6 +66,7 @@ const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
full: false,
historyLimit: 12,
saveSnapshot: true,
memoryPressureOnly: false,
};
const GC_CONFIG_RELATIVE_PATH = "config/unidesk-cli.yaml";
@@ -89,7 +91,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
return {
ok: false,
error: "gc-remote-provider-required",
usage: "bun scripts/cli.ts gc remote <providerId> plan|snapshot|trend|run|status|policy [--confirm]",
usage: "bun scripts/cli.ts gc remote <providerId> memory|plan|snapshot|trend|run|status|policy [--confirm]",
};
}
const subaction = action ?? "plan";
@@ -124,6 +126,17 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
};
}
const options = parseRemoteGcOptions(args);
if (options.memoryPressureOnly && subaction !== "plan" && subaction !== "run") {
return {
ok: false,
error: "gc-remote-memory-pressure-only-action-invalid",
action: subaction,
supportedActions: ["plan", "run"],
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan --memory-pressure-only`,
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm --memory-pressure-only`,
};
}
if (subaction === "memory") return await runRemoteGc(config, providerId, "memory", options);
if (subaction === "plan" || subaction === "dry-run") return await runRemoteGc(config, providerId, "plan", options);
if (subaction === "snapshot" || subaction === "growth") return await runRemoteGc(config, providerId, "snapshot", options);
if (subaction === "trend") return await runRemoteGc(config, providerId, "trend", options);
@@ -136,8 +149,8 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
dryRun: true,
mutation: false,
requiredFlag: "--confirm",
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan`,
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm`,
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`,
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`,
};
}
return await runRemoteGc(config, providerId, "run", options);
@@ -146,7 +159,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
ok: false,
error: "unsupported-gc-remote-action",
action: subaction,
supportedActions: ["plan", "snapshot", "trend", "run", "status", "policy"],
supportedActions: ["memory", "plan", "snapshot", "trend", "run", "status", "policy"],
};
}
@@ -238,6 +251,8 @@ function parseRemoteGcOptions(args: string[]): RemoteGcOptions {
options.full = true;
} else if (arg === "--no-save" || arg === "--no-snapshot-save") {
options.saveSnapshot = false;
} else if (arg === "--memory-pressure-only") {
options.memoryPressureOnly = true;
} else {
throw new Error(`unknown gc remote option: ${arg}`);
}
+5
View File
@@ -517,6 +517,10 @@ function gcHelp(): unknown {
"bun scripts/cli.ts gc policy plan",
"bun scripts/cli.ts gc policy install",
"bun scripts/cli.ts gc remote G14 plan",
"bun scripts/cli.ts gc remote NC01 memory",
"bun scripts/cli.ts gc remote NC01 plan --memory-pressure-only",
"bun scripts/cli.ts gc remote NC01 run --confirm --memory-pressure-only",
"bun scripts/cli.ts gc remote NC01 status --job-id <job-id>",
"bun scripts/cli.ts gc remote G14 plan --target-use-percent 50 --include-hwlab-registry",
"bun scripts/cli.ts gc remote G14 run --confirm",
"bun scripts/cli.ts gc remote G14 status --job-id <id>",
@@ -563,6 +567,7 @@ function gcHelp(): unknown {
"db-trace run --vacuum-full": "rewrite public.oa_events after deletion so df can reclaim disk; requires maintenance window",
"policy plan|install": "render or install journald caps and a daily low-risk gc systemd timer from config/unidesk-cli.yaml#gc.policyTimer",
"remote <providerId> plan|run": "run bounded GC through UniDesk SSH passthrough on a provider host; G14 protects HWLAB k3s/runtime/PVC/workspace paths, and HWLAB registry retention is explicit opt-in with workload-ref, digest-closure, recent-tag and per-repo tag protection",
"remote <providerId> memory": "只读输出目标节点 /proc/meminfo 的 MemAvailable;不合并 swap、cgroup 或 virtual memory,也不执行 GC mutation",
"--no-file-logs|--no-docker-logs|--no-journal|--no-build-cache|--no-tmp|--no-db-summary": "disable one collector",
},
reference: "docs/reference/gc.md",
+117
View File
@@ -148,6 +148,7 @@ export interface HwlabRuntimeWebProbeSpec {
readonly browserProxyMode?: "auto" | "direct";
readonly playwrightBrowsersPath?: string;
readonly defaultOrigin?: HwlabRuntimeWebProbeOriginName;
readonly resourcePolicy?: HwlabRuntimeWebProbeResourcePolicySpec;
readonly origins?: HwlabRuntimeWebProbeOriginsSpec;
readonly authLogin?: HwlabRuntimeWebProbeAuthLoginSpec;
readonly alertThresholds?: HwlabRuntimeWebProbeAlertThresholdsSpec;
@@ -158,6 +159,61 @@ export interface HwlabRuntimeWebProbeSpec {
readonly workbenchTraceReadability?: HwlabRuntimeWebProbeWorkbenchTraceReadabilitySpec;
}
export interface HwlabRuntimeWebProbeMemoryStartGuardSpec {
readonly metric: "MemAvailable";
readonly source: "/proc/meminfo";
readonly swapExcluded: true;
readonly thresholdBytes: number;
readonly manualComparator: "less-than-or-equal";
readonly sentinelComparator: "less-than";
readonly snapshotTimeoutSeconds: number;
readonly launchLockPath: string;
readonly launchLockTimeoutSeconds: number;
readonly launchReadyTimeoutSeconds: number;
readonly launchReadyPollMilliseconds: number;
}
export interface HwlabRuntimeWebProbeObserverLifecycleSpec {
readonly maxRunSeconds: number;
readonly maxSamples: number;
readonly maxScreenshots: number;
readonly maxPerformanceEntries: number;
readonly maxBrowserProcessHistoryEntries: number;
readonly maxBrowserFreezeSignalHistoryEntries: number;
readonly maxRealtimeEventEntries: number;
readonly maxRealtimeEventBodyBytes: number;
readonly maxDomEvidenceRows: number;
readonly maxProcessScanEntries: number;
readonly maxPerformanceCaptureSeconds: number;
readonly maxResponseBodyBytes: number;
readonly maxWebPerformanceBodyBytes: number;
readonly maxJsonlQueueEntries: number;
readonly maxJsonlQueueBytes: number;
readonly maxJsonlLineBytes: number;
readonly jsonlFlushTimeoutMs: number;
readonly maxResponseBodyInFlight: number;
readonly maxPassiveListenerTasks: number;
readonly passiveTaskFlushTimeoutMs: number;
readonly closeStepTimeoutMs: number;
readonly maxProcessReadConcurrency: number;
readonly maxRealtimeTotalEntries: number;
readonly maxRealtimeTotalBodyBytes: number;
}
export interface HwlabRuntimeWebProbeResourcePolicySpec {
readonly memoryStartGuard: HwlabRuntimeWebProbeMemoryStartGuardSpec;
readonly observerLifecycle: HwlabRuntimeWebProbeObserverLifecycleSpec;
readonly browserEvidence: HwlabRuntimeWebProbeBrowserEvidenceSpec;
}
export interface HwlabRuntimeWebProbeBrowserEvidenceSpec {
readonly screenshotMode: "viewport";
readonly maxNetworkRequestEntries: number;
readonly maxNetworkRequestBytes: number;
readonly maxNetworkFailureEntries: number;
readonly maxNetworkFailureBytes: number;
}
export interface HwlabRuntimeWebProbeWorkbenchKafkaDebugReplaySpec {
readonly enabled: boolean;
readonly topic: string;
@@ -754,6 +810,11 @@ function booleanField(obj: Record<string, unknown>, key: string, path: string):
return value;
}
function literalTrueField(obj: Record<string, unknown>, key: string, path: string): true {
if (obj[key] !== true) throw new Error(`${path}.${key} must be true`);
return true;
}
function nonNegativeIntegerField(obj: Record<string, unknown>, key: string, path: string): number {
const value = obj[key];
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`);
@@ -1410,6 +1471,7 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec
...(browserProxyMode === undefined ? {} : { browserProxyMode }),
...(playwrightBrowsersPath === undefined ? {} : { playwrightBrowsersPath }),
...(defaultOrigin === undefined ? {} : { defaultOrigin }),
...(raw.resourcePolicy === undefined ? {} : { resourcePolicy: webProbeResourcePolicyConfig(raw.resourcePolicy, `${path}.resourcePolicy`) }),
...(origins === undefined ? {} : { origins }),
...(raw.authLogin === undefined ? {} : { authLogin: webProbeAuthLoginConfig(raw.authLogin, `${path}.authLogin`) }),
...(raw.alertThresholds === undefined ? {} : { alertThresholds: webProbeAlertThresholdsConfig(raw.alertThresholds, `${path}.alertThresholds`) }),
@@ -1425,6 +1487,61 @@ function webProbeConfig(value: unknown, path: string): HwlabRuntimeWebProbeSpec
};
}
function webProbeResourcePolicyConfig(value: unknown, path: string): HwlabRuntimeWebProbeResourcePolicySpec {
const raw = asRecord(value, path);
const memory = asRecord(raw.memoryStartGuard, `${path}.memoryStartGuard`);
const lifecycle = asRecord(raw.observerLifecycle, `${path}.observerLifecycle`);
const browserEvidence = asRecord(raw.browserEvidence, `${path}.browserEvidence`);
return {
memoryStartGuard: {
metric: enumStringField(memory, "metric", `${path}.memoryStartGuard`, ["MemAvailable"] as const),
source: enumStringField(memory, "source", `${path}.memoryStartGuard`, ["/proc/meminfo"] as const),
swapExcluded: literalTrueField(memory, "swapExcluded", `${path}.memoryStartGuard`),
thresholdBytes: boundedIntegerField(memory, "thresholdBytes", `${path}.memoryStartGuard`, 1, Number.MAX_SAFE_INTEGER),
manualComparator: enumStringField(memory, "manualComparator", `${path}.memoryStartGuard`, ["less-than-or-equal"] as const),
sentinelComparator: enumStringField(memory, "sentinelComparator", `${path}.memoryStartGuard`, ["less-than"] as const),
snapshotTimeoutSeconds: boundedIntegerField(memory, "snapshotTimeoutSeconds", `${path}.memoryStartGuard`, 5, 120),
launchLockPath: absoluteHostPathField(stringField(memory, "launchLockPath", `${path}.memoryStartGuard`), `${path}.memoryStartGuard.launchLockPath`),
launchLockTimeoutSeconds: boundedIntegerField(memory, "launchLockTimeoutSeconds", `${path}.memoryStartGuard`, 1, 60),
launchReadyTimeoutSeconds: boundedIntegerField(memory, "launchReadyTimeoutSeconds", `${path}.memoryStartGuard`, 1, 30),
launchReadyPollMilliseconds: boundedIntegerField(memory, "launchReadyPollMilliseconds", `${path}.memoryStartGuard`, 50, 1000),
},
observerLifecycle: {
maxRunSeconds: boundedIntegerField(lifecycle, "maxRunSeconds", `${path}.observerLifecycle`, 1, 86_400),
maxSamples: boundedIntegerField(lifecycle, "maxSamples", `${path}.observerLifecycle`, 1, 10_000_000),
maxScreenshots: boundedIntegerField(lifecycle, "maxScreenshots", `${path}.observerLifecycle`, 1, 10_000),
maxPerformanceEntries: boundedIntegerField(lifecycle, "maxPerformanceEntries", `${path}.observerLifecycle`, 1, 100_000),
maxBrowserProcessHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserProcessHistoryEntries", `${path}.observerLifecycle`, 1, 100_000),
maxBrowserFreezeSignalHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserFreezeSignalHistoryEntries", `${path}.observerLifecycle`, 1, 100_000),
maxRealtimeEventEntries: boundedIntegerField(lifecycle, "maxRealtimeEventEntries", `${path}.observerLifecycle`, 1, 100_000),
maxRealtimeEventBodyBytes: boundedIntegerField(lifecycle, "maxRealtimeEventBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
maxDomEvidenceRows: boundedIntegerField(lifecycle, "maxDomEvidenceRows", `${path}.observerLifecycle`, 1, 100_000),
maxProcessScanEntries: boundedIntegerField(lifecycle, "maxProcessScanEntries", `${path}.observerLifecycle`, 1, 1_000_000),
maxPerformanceCaptureSeconds: boundedIntegerField(lifecycle, "maxPerformanceCaptureSeconds", `${path}.observerLifecycle`, 1, 600),
maxResponseBodyBytes: boundedIntegerField(lifecycle, "maxResponseBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
maxWebPerformanceBodyBytes: boundedIntegerField(lifecycle, "maxWebPerformanceBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
maxJsonlQueueEntries: boundedIntegerField(lifecycle, "maxJsonlQueueEntries", `${path}.observerLifecycle`, 1, 100_000),
maxJsonlQueueBytes: boundedIntegerField(lifecycle, "maxJsonlQueueBytes", `${path}.observerLifecycle`, 1024, 64 * 1024 * 1024),
maxJsonlLineBytes: boundedIntegerField(lifecycle, "maxJsonlLineBytes", `${path}.observerLifecycle`, 1024, 16 * 1024 * 1024),
jsonlFlushTimeoutMs: boundedIntegerField(lifecycle, "jsonlFlushTimeoutMs", `${path}.observerLifecycle`, 100, 60_000),
maxResponseBodyInFlight: boundedIntegerField(lifecycle, "maxResponseBodyInFlight", `${path}.observerLifecycle`, 1, 64),
maxPassiveListenerTasks: boundedIntegerField(lifecycle, "maxPassiveListenerTasks", `${path}.observerLifecycle`, 1, 10_000),
passiveTaskFlushTimeoutMs: boundedIntegerField(lifecycle, "passiveTaskFlushTimeoutMs", `${path}.observerLifecycle`, 100, 60_000),
closeStepTimeoutMs: boundedIntegerField(lifecycle, "closeStepTimeoutMs", `${path}.observerLifecycle`, 100, 60_000),
maxProcessReadConcurrency: boundedIntegerField(lifecycle, "maxProcessReadConcurrency", `${path}.observerLifecycle`, 1, 256),
maxRealtimeTotalEntries: boundedIntegerField(lifecycle, "maxRealtimeTotalEntries", `${path}.observerLifecycle`, 1, 100_000),
maxRealtimeTotalBodyBytes: boundedIntegerField(lifecycle, "maxRealtimeTotalBodyBytes", `${path}.observerLifecycle`, 1024, 64 * 1024 * 1024),
},
browserEvidence: {
screenshotMode: enumStringField(browserEvidence, "screenshotMode", `${path}.browserEvidence`, ["viewport"] as const),
maxNetworkRequestEntries: boundedIntegerField(browserEvidence, "maxNetworkRequestEntries", `${path}.browserEvidence`, 1, 100_000),
maxNetworkRequestBytes: boundedIntegerField(browserEvidence, "maxNetworkRequestBytes", `${path}.browserEvidence`, 1024, 64 * 1024 * 1024),
maxNetworkFailureEntries: boundedIntegerField(browserEvidence, "maxNetworkFailureEntries", `${path}.browserEvidence`, 1, 100_000),
maxNetworkFailureBytes: boundedIntegerField(browserEvidence, "maxNetworkFailureBytes", `${path}.browserEvidence`, 1024, 64 * 1024 * 1024),
},
};
}
function webProbeWorkbenchKafkaDebugReplayConfig(value: unknown, path: string): HwlabRuntimeWebProbeWorkbenchKafkaDebugReplaySpec {
const raw = asRecord(value, path);
const topic = stringField(raw, "topic", path);
@@ -379,7 +379,7 @@ async function authenticate(browserContext) {
await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item?.retryLabel || retryLabel, retryAttempt: attempt, retryMaxAttempts: maxAttempts, retryDelayMs: item?.retryDelayMs ?? 0, lastStatus: item?.status ?? 0, lastStatusText: item?.statusText ?? "request-error", retryable, cookiePresent: false, retryExhausted: false, lastError: item?.error || null, valuesRedacted: true } }).catch(() => {});
if (!retryable) break;
}
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs);
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await interruptibleRunnerSleep(retryDelayMs);
}
const cookieState = await readAuthCookieState(browserContext);
const last = attempts[attempts.length - 1] || null;
@@ -414,12 +414,13 @@ async function pageAuthLogin(browserContext, loginUrl, credential = { username,
headers: { accept: "application/json", "content-type": "application/json" },
timeout: authLoginRequestTimeoutMs,
});
await response.text().catch(() => "");
return {
const result = {
ok: response.ok(),
status: response.status(),
statusText: response.statusText() || "",
};
if (typeof response.dispose === "function") await response.dispose().catch(() => {});
return result;
}
async function loginAccount(command) {
@@ -479,7 +480,7 @@ async function loginAccount(command) {
cookieState = await readAuthCookieState(context).catch(() => ({ cookiePresent: false, cookieNames: [] }));
if (!retryable) break;
}
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs);
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await interruptibleRunnerSleep(retryDelayMs);
}
response = response ?? { ok: false, status: 0, statusText: "api-login-failed" };
cookieState = cookieState ?? await readAuthCookieState(context);
@@ -514,7 +515,6 @@ async function logoutAccount(command = {}) {
const logoutUrl = new URL("/logout", baseUrl).toString();
const response = await page.evaluate(async (input) => {
const res = await fetch(input.logoutUrl, { method: "POST", headers: { accept: "application/json" }, credentials: "include" });
await res.text().catch(() => "");
return { ok: res.ok, status: res.status, statusText: res.statusText || "" };
}, { logoutUrl });
await context.clearCookies().catch(() => {});
@@ -1295,11 +1295,9 @@ async function createSessionFromUi() {
const createStatus = createResponse.status();
let createPayload = null;
let createPayloadError = null;
try {
createPayload = await createResponse.json();
} catch (error) {
createPayloadError = errorSummary(error);
}
const createBody = await boundedResponseJson(createResponse);
createPayload = createBody.value;
createPayloadError = createBody.error;
const createdSessionId = sessionIdFromAgentSessionPayload(createPayload);
if (createStatus < 200 || createStatus >= 300 || !createdSessionId) {
const error = new Error("newSession did not receive an authoritative session id from POST /v1/agent/sessions");
@@ -1753,11 +1751,9 @@ async function sendPrompt(text, options = {}) {
}
let chatPayload = null;
let chatPayloadError = null;
try {
chatPayload = await chatResponse.json();
} catch (error) {
chatPayloadError = errorSummary(error);
}
const chatBody = await boundedResponseJson(chatResponse);
chatPayload = chatBody.value;
chatPayloadError = chatBody.error;
const payloadText = chatPayload ? JSON.stringify(chatPayload) : "";
const traceId = payloadText.match(/\btrc_[A-Za-z0-9_-]+\b/u)?.[0] || null;
const otelTraceId = typeof chatPayload?.otelTrace?.traceId === "string" && /^[0-9a-f]{32}$/u.test(chatPayload.otelTrace.traceId)
@@ -148,6 +148,7 @@ async function validateWorkbenchKafkaDebugReplay(command) {
try { url = new URL(request.url()); } catch { return; }
if (url.pathname === config.productEventsPath) {
productEventSourceRequests.push({ method: request.method(), path: url.pathname, phase: requestPhase, valuesRedacted: true });
if (productEventSourceRequests.length > maxRealtimeEventEntries) productEventSourceRequests.splice(0, productEventSourceRequests.length - maxRealtimeEventEntries);
return;
}
if (url.pathname !== config.debugEventsPath) return;
@@ -160,6 +161,7 @@ async function validateWorkbenchKafkaDebugReplay(command) {
phase: requestPhase,
valuesRedacted: true
});
if (requests.length > maxRealtimeEventEntries) requests.splice(0, requests.length - maxRealtimeEventEntries);
};
try {
const controlRecovery = await ensureControlPageResponsiveForCommand("validateWorkbenchKafkaDebugReplay");
@@ -386,7 +388,7 @@ async function workbenchKafkaDebugReplayWaitForCleared(panel, countsNode, timeou
if (status === "idle" && counts?.appliedCount === 0 && counts?.receivedCount === 0) {
return { status, ...counts, valuesRedacted: true };
}
await sleep(100);
await interruptibleRunnerSleep(100);
}
throw new Error("Workbench Kafka debug panel did not clear to idle 0/0");
}
@@ -397,7 +399,7 @@ async function workbenchKafkaDebugReplayWaitForTerminal(panel, timeoutMs) {
const status = await panel.getAttribute("data-replay-status");
if (status === "completed") return status;
if (status === "incomplete" || status === "error" || status === "closed") return status;
await sleep(200);
await interruptibleRunnerSleep(200);
}
throw new Error("Workbench Kafka debug replay did not reach a terminal panel status within " + timeoutMs + "ms");
}
@@ -406,7 +408,7 @@ async function workbenchKafkaDebugReplayWaitForTabSelected(tab, timeoutMs) {
const deadline = Date.now() + timeoutMs;
while (Date.now() <= deadline) {
if (await tab.getAttribute("aria-selected") === "true") return true;
await sleep(50);
await interruptibleRunnerSleep(50);
}
return false;
}
@@ -514,7 +516,7 @@ async function workbenchKafkaDebugReplayWaitForStablePanel(panel, countsNode, ti
stableSinceMs = null;
}
previousSignature = signature;
await sleep(100);
await interruptibleRunnerSleep(100);
}
return {
...latest,
@@ -586,14 +588,14 @@ async function workbenchKafkaDebugReplayInspectRawWindow(input) {
await rawCountsNode.waitFor({ state: "visible", timeout: 5000 });
await rawTextNode.waitFor({ state: "visible", timeout: 5000 });
await rawContractNode.waitFor({ state: "visible", timeout: 5000 });
await sleep(input.config.requestObservationQuietMs);
await interruptibleRunnerSleep(input.config.requestObservationQuietMs);
const counts = workbenchKafkaDebugReplayRawCounts(await rawCountsNode.textContent());
const rawText = await rawTextNode.inputValue();
const contractText = String(await rawContractNode.textContent() || "").replace(/\s+/gu, " ").trim();
input.setRequestPhase("trace-tab-restore");
await traceTab.click({ timeout: 10000 });
if (!await workbenchKafkaDebugReplayWaitForTabSelected(traceTab, 5000)) throw new Error("Workbench Trace debug tab did not restore after raw inspection");
await sleep(input.config.requestObservationQuietMs);
await interruptibleRunnerSleep(input.config.requestObservationQuietMs);
input.setRequestPhase("complete");
const pageUrlAfter = currentPageUrl();
const productRequests = input.productEventSourceRequests.slice(requestStart);
@@ -16,7 +16,7 @@ async function installPagePerformanceProbe(targetPage, pageRole = "control", tar
}
const buffer = [];
const supportedTypes = [];
const maxBufferSize = 2000;
const maxBufferSize = Math.max(1, Number(input.maxBufferSize || 1));
const push = (event) => {
buffer.push({
probeVersion: 1,
@@ -126,9 +126,9 @@ async function installPagePerformanceProbe(targetPage, pageRole = "control", tar
};
return { ok: true, alreadyInstalled: false, supportedTypes, valuesRedacted: true };
};
await targetPage.addInitScript(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs });
await targetPage.addInitScript(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs, maxBufferSize: maxPerformanceEntries });
if (!targetPage.isClosed()) {
await withHardTimeout(targetPage.evaluate(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs }), 3000, "installPagePerformanceProbe evaluate exceeded 3000ms");
await withHardTimeout(targetPage.evaluate(installer, { pageRole, pageId: targetPageId, pageEpoch: pageRole === "observer" ? observerPageEpoch : controlPageEpoch, eventLoopGapRedMs: alertThresholds.eventLoopGapRedMs, maxBufferSize: maxPerformanceEntries }), 3000, "installPagePerformanceProbe evaluate exceeded 3000ms");
}
}
@@ -227,7 +227,7 @@ function compactPerformanceAttribution(item) {
}
async function capturePerformanceProfile(command) {
const durationMs = boundedInteger(command.durationMs, 5000, 100, 600000);
const durationMs = boundedInteger(command.durationMs, Math.min(5000, maxPerformanceCaptureMs), 100, maxPerformanceCaptureMs);
const timeoutMs = Math.max(3000, Math.min(15000, Number(alertThresholds.playwrightResponsivenessRedMs) || 5000));
const captureId = safeId(command.label || command.id || ("perf-" + Date.now().toString(36))) || ("perf-" + Date.now().toString(36));
const targetPage = page && !page.isClosed() ? page : observerPage && !observerPage.isClosed() ? observerPage : null;
@@ -264,7 +264,7 @@ async function capturePerformanceProfile(command) {
beforeDrain,
valuesRedacted: true,
}));
await sleep(durationMs);
await interruptibleRunnerSleep(durationMs);
stopped = await withHardTimeout(session.send("Profiler.stop"), Math.max(timeoutMs, 5000), "Profiler.stop exceeded " + Math.max(timeoutMs, 5000) + "ms");
} catch (error) {
captureError = error;
@@ -8,14 +8,26 @@ test("realtime fanout keeps the YAML terminal timeout when durationMs is absent"
const build = new Function(
"boundedInteger",
"sanitize",
"maxRealtimeTotalEntries",
"maxRealtimeTotalBodyBytes",
"maxRealtimeEventEntries",
"maxRealtimeEventBodyBytes",
`${source}\nreturn realtimeFanoutEffectiveProfile;`,
) as (
boundedInteger: (value: unknown, fallback: number, min: number, max: number) => number,
sanitize: <T>(value: T) => T,
maxRealtimeTotalEntries: number,
maxRealtimeTotalBodyBytes: number,
maxRealtimeEventEntries: number,
maxRealtimeEventBodyBytes: number,
) => (profile: Record<string, unknown>, durationMs: unknown) => Record<string, unknown>;
const effectiveProfile = build(
(value, fallback, min, max) => Math.min(max, Math.max(min, Number.isFinite(Number(value)) ? Number(value) : fallback)),
(value) => value,
512,
4 * 1024 * 1024,
256,
64 * 1024,
);
const profile = {
subscriberCount: 2,
@@ -48,6 +60,9 @@ test("realtime fanout keeps the YAML terminal timeout when durationMs is absent"
assert.equal(effectiveProfile(profile, undefined).terminalTimeoutMs, 480_000);
assert.equal(effectiveProfile(profile, 5_000).terminalTimeoutMs, 5_000);
assert.deepEqual(effectiveProfile(profile, null).conditionalEventTypePairs, profile.conditionalEventTypePairs);
const budget = effectiveProfile(profile, null).aggregateBudget as Record<string, number>;
assert.ok(budget.perArrayEntries * budget.entrySlots <= budget.maxTotalEntries);
assert.ok(budget.perEventBodyBytes * budget.bodyRetentionSlots <= budget.maxTotalBodyBytes);
});
test("realtime fanout terminal completion does not require absent optional tool families", () => {
@@ -137,12 +152,12 @@ test("realtime fanout validates optional tool events only when they occur", () =
test("realtime fanout retries a YAML-bounded transient health navigation", async () => {
const source = nodeWebObserveRunnerRealtimeSource();
const build = new Function(
"sleep",
"interruptibleRunnerSleep",
"isRetryableNavigationError",
"baseUrl",
`${source}\nreturn realtimeGotoHealth;`,
) as (
sleep: (milliseconds: number) => Promise<void>,
interruptibleRunnerSleep: (milliseconds: number) => Promise<void>,
isRetryableNavigationError: (message: string) => boolean,
baseUrl: string,
) => (page: { goto: () => Promise<{ status: () => number }> }, timeoutMs: number, profile: Record<string, number>, label: string) => Promise<Record<string, unknown>>;
@@ -433,11 +448,11 @@ test("realtime replay joint quiet waits for a late formal Kafka envelope before
let kafkaRows = [first];
const productRows = [first, late];
const build = new Function(
"sleep",
"interruptibleRunnerSleep",
"Date",
`${source}\nreturn realtimeWaitReplayedTraceJointQuiet;`,
) as (
sleep: () => Promise<void>,
interruptibleRunnerSleep: () => Promise<void>,
date: { now: () => number },
) => (
debug: Record<string, any>,
@@ -495,11 +510,11 @@ test("realtime replay joint quiet rejects a permanent 31/32 split with bounded i
const first = replayEnvelope("evt_first", 1);
const late = replayEnvelope("evt_late", 2, true);
const build = new Function(
"sleep",
"interruptibleRunnerSleep",
"Date",
`${source}\nreturn realtimeWaitReplayedTraceJointQuiet;`,
) as (
sleep: () => Promise<void>,
interruptibleRunnerSleep: () => Promise<void>,
date: { now: () => number },
) => (
debug: Record<string, any>,
@@ -23,6 +23,8 @@ async function validateRealtimeFanout(command) {
if (!provider) throw new Error("validateRealtimeFanout requires provider");
if (!firstPrompt || !secondPrompt) throw new Error("validateRealtimeFanout requires first and second prompt");
const effective = realtimeFanoutEffectiveProfile(profile, command.durationMs);
const aggregateCounters = { networkDroppedEntries: 0, valuesRedacted: true };
realtimeAggregateLast = { commandId: command.id, profile: profileId, budget: effective.aggregateBudget, counters: aggregateCounters, status: "running", valuesRedacted: true };
const reportDir = path.join(stateDir, "artifacts", "realtime-fanout", safeId(command.id));
await mkdir(reportDir, { recursive: true, mode: 0o700 });
const network = [];
@@ -40,7 +42,10 @@ async function validateRealtimeFanout(command) {
lastEventId: request.headers()["last-event-id"] || null,
valuesRedacted: true,
});
if (network.length > 1000) network.splice(0, network.length - 1000);
if (network.length > effective.aggregateBudget.perArrayEntries) {
aggregateCounters.networkDroppedEntries += network.length - effective.aggregateBudget.perArrayEntries;
network.splice(0, network.length - effective.aggregateBudget.perArrayEntries);
}
};
page.on("request", requestListener);
let subscriberA = null;
@@ -130,7 +135,7 @@ async function validateRealtimeFanout(command) {
subscriberB2 = await realtimeNewSubscriber(storageState, sessionId, "subscriber-b-reconnected", effective);
const bReconnected = await realtimeWaitProductConnected(subscriberB2, effective.barrierTimeoutMs);
realtimeAssertConnectedContract(bReconnected, sessionId, "subscriber-b-reconnected", effective, { requireRetainedEvents: effective.expectedProductSse.replayPriorTraceOnReconnect });
await sleep(effective.reconnectQuietMs);
await interruptibleRunnerSleep(effective.reconnectQuietMs);
let reconnectBeforeSecond;
if (effective.expectedProductSse.replayPriorTraceOnReconnect) {
const firstReplayBeforeSecond = await realtimeWaitReplayedTraceJointQuiet(
@@ -258,6 +263,13 @@ async function validateRealtimeFanout(command) {
realtimeTurnSummary(first, firstDebug, productA, firstTerminal),
realtimeTurnSummary(second, secondDebug, productB2, secondTerminalB),
];
aggregateCounters.productDroppedEntries = Number(productA.droppedEntries || 0) + Number(productB2.droppedEntries || 0);
aggregateCounters.debugDroppedEntries = Number(firstDebug.droppedEntries || 0) + Number(secondDebug.droppedEntries || 0);
aggregateCounters.productDroppedBodyEntries = Number(productA.droppedBodyEntries || 0) + Number(productB2.droppedBodyEntries || 0);
aggregateCounters.productDroppedBodyBytes = Number(productA.droppedBodyBytes || 0) + Number(productB2.droppedBodyBytes || 0);
aggregateCounters.debugDroppedBodyEntries = Number(firstDebug.droppedBodyEntries || 0) + Number(secondDebug.droppedBodyEntries || 0);
aggregateCounters.debugDroppedBodyBytes = Number(firstDebug.droppedBodyBytes || 0) + Number(secondDebug.droppedBodyBytes || 0);
realtimeAggregateLast = { ...realtimeAggregateLast, status: "completed", counters: aggregateCounters, valuesRedacted: true };
Object.assign(report, {
ok: true,
status: "passed",
@@ -290,6 +302,7 @@ async function validateRealtimeFanout(command) {
valuesRedacted: true,
},
forbiddenRequests: { count: forbidden.length, rows: forbidden, valuesRedacted: true },
aggregateBudget: { ...effective.aggregateBudget, counters: aggregateCounters, valuesRedacted: true },
screenshots: [bBeforeShot, summaryScreenshot, subscriberAScreenshot, subscriberB2Screenshot],
warmRunnerReused: first.runnerId && second.runnerId ? first.runnerId === second.runnerId : null,
valuesRedacted: true,
@@ -326,6 +339,7 @@ async function validateRealtimeFanout(command) {
valuesRedacted: true,
};
} catch (error) {
realtimeAggregateLast = { ...realtimeAggregateLast, status: "failed", counters: aggregateCounters, failure: errorSummary(error), valuesRedacted: true };
Object.assign(report, {
ok: false,
status: "failed",
@@ -367,6 +381,8 @@ async function validateExistingSessionRefresh(command) {
const profile = realtimeFanoutProfiles[profileId];
if (!profile || typeof profile !== "object") throw new Error("validateExistingSessionRefresh profile is not declared by the owning YAML: " + profileId);
const effective = realtimeFanoutEffectiveProfile(profile, command.durationMs);
const aggregateCounters = { networkDroppedEntries: 0, valuesRedacted: true };
realtimeAggregateLast = { commandId: command.id, profile: profileId, budget: effective.aggregateBudget, counters: aggregateCounters, status: "running", valuesRedacted: true };
if (effective.expectedProductSse.refreshHandoff !== true) throw new Error("validateExistingSessionRefresh requires a YAML profile with refresh handoff enabled");
const reportDir = path.join(stateDir, "artifacts", "existing-session-refresh", safeId(command.id));
await mkdir(reportDir, { recursive: true, mode: 0o700 });
@@ -385,7 +401,10 @@ async function validateExistingSessionRefresh(command) {
lastEventId: request.headers()["last-event-id"] || null,
valuesRedacted: true,
});
if (network.length > 1000) network.splice(0, network.length - 1000);
if (network.length > effective.aggregateBudget.perArrayEntries) {
aggregateCounters.networkDroppedEntries += network.length - effective.aggregateBudget.perArrayEntries;
network.splice(0, network.length - effective.aggregateBudget.perArrayEntries);
}
};
const report = {
ok: false,
@@ -447,7 +466,7 @@ async function validateExistingSessionRefresh(command) {
}
phase = "live-handoff";
await realtimeWaitForSameWorkbenchDom(before, effective.barrierTimeoutMs);
await sleep(effective.reconnectQuietMs);
await interruptibleRunnerSleep(effective.reconnectQuietMs);
const after = await realtimeWorkbenchDomIdentity();
realtimeAssertSameWorkbenchDom(before, after);
const browserEvidence = await page.evaluate(() => JSON.parse(JSON.stringify(window.__unideskExistingSessionRefresh || {})));
@@ -463,11 +482,16 @@ async function validateExistingSessionRefresh(command) {
if (mainStreams[0].traceId !== null || mainStreams[0].afterSeq !== null || mainStreams[0].lastEventId !== null) throw new Error("main Workbench product SSE used a trace or cursor scope");
if (forbidden.length > 0) throw new Error("forbidden recovery or polling requests observed: " + forbidden.map((item) => item.path).join(","));
phase = "evidence-write";
const screenshotFile = path.join(reportDir, "existing-session-refresh.png");
await page.screenshot({ path: screenshotFile, fullPage: true, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
const screenshotMeta = await fileMeta(screenshotFile);
screenshot = { kind: "existing-session-refresh-screenshot", path: screenshotFile, ...screenshotMeta, commandId: activeCommandId, valuesRedacted: true };
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...screenshot });
const screenshotSkipped = reserveScreenshot("existing-session-refresh");
if (screenshotSkipped !== null) {
screenshot = screenshotSkipped;
} else {
const screenshotFile = path.join(reportDir, "existing-session-refresh.png");
await page.screenshot({ path: screenshotFile, fullPage: false, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
const screenshotMeta = await fileMeta(screenshotFile);
screenshot = { kind: "existing-session-refresh-screenshot", path: screenshotFile, ...screenshotMeta, commandId: activeCommandId, valuesRedacted: true };
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...screenshot });
}
const productSse = {
requestCount: mainStreams.length,
browserSourceCount: browserEvidence.sources.length,
@@ -488,10 +512,12 @@ async function validateExistingSessionRefresh(command) {
connected: connectedSummary,
refreshReplay,
productSse,
aggregateBudget: { ...effective.aggregateBudget, counters: { ...aggregateCounters, browserDroppedEntries: Number(browserEvidence.droppedEntries || 0), browserDroppedBodyEntries: Number(browserEvidence.droppedBodyEntries || 0), browserDroppedBodyBytes: Number(browserEvidence.droppedBodyBytes || 0) }, valuesRedacted: true },
forbiddenRequests: { count: 0, rows: [], valuesRedacted: true },
screenshot,
valuesRedacted: true,
});
realtimeAggregateLast = { ...realtimeAggregateLast, status: "completed", counters: { ...aggregateCounters, browserDroppedEntries: Number(browserEvidence.droppedEntries || 0), browserDroppedBodyEntries: Number(browserEvidence.droppedBodyEntries || 0), browserDroppedBodyBytes: Number(browserEvidence.droppedBodyBytes || 0) }, valuesRedacted: true };
const artifacts = await realtimeWriteEvidenceArtifacts(reportDir, network, [], "existing-session-refresh");
report.artifacts = artifacts;
const reportArtifact = await realtimeWriteReport(reportDir, report, "existing-session-refresh-report");
@@ -527,6 +553,7 @@ async function validateExistingSessionRefresh(command) {
valuesRedacted: true,
};
} catch (error) {
realtimeAggregateLast = { ...realtimeAggregateLast, status: "failed", counters: aggregateCounters, failure: errorSummary(error), valuesRedacted: true };
const failure = realtimeExistingRefreshFailure(phase, error);
if (!screenshot) {
screenshot = await captureScreenshot("existing-session-refresh-failed-" + safeId(command.id))
@@ -649,10 +676,17 @@ async function realtimeArmExistingSessionRefreshProbe(targetPage, profile) {
let config;
try { config = JSON.parse(raw); } catch { return; }
const NativeEventSource = window.EventSource;
const state = { sources: [], connected: [], businessEventCount: 0, failures: [], errors: [] };
const state = { sources: [], connected: [], businessEventCount: 0, failures: [], errors: [], droppedEntries: 0, droppedBodyEntries: 0, droppedBodyBytes: 0 };
const maxEntries = Math.max(1, Number(config.maxEntries || 1));
const maxEventTextChars = Math.max(1, Number(config.maxEventTextChars || 1));
const bodyBytes = (value) => new TextEncoder().encode(String(value || "")).byteLength;
const pushBounded = (items, value) => {
items.push(value);
if (items.length > maxEntries) { state.droppedEntries += items.length - maxEntries; items.splice(0, items.length - maxEntries); }
};
window.__unideskExistingSessionRefresh = state;
if (typeof NativeEventSource !== "function") {
state.errors.push({ kind: "eventsource-unavailable" });
pushBounded(state.errors, { kind: "eventsource-unavailable" });
return;
}
class ObservedEventSource extends NativeEventSource {
@@ -661,17 +695,23 @@ async function realtimeArmExistingSessionRefreshProbe(targetPage, profile) {
let pathname = "";
try { pathname = new URL(String(url), window.location.href).pathname; } catch {}
if (pathname !== config.productEventsPath) return;
state.sources.push({ url: String(url), opened: 0 });
pushBounded(state.sources, { url: String(url), opened: 0 });
const source = state.sources[state.sources.length - 1];
this.addEventListener("open", () => { source.opened += 1; });
this.addEventListener(config.productReadyEvent, (event) => {
try { state.connected.push(JSON.parse(event.data)); } catch { state.errors.push({ kind: "connected-parse" }); }
const raw = String(event.data || "");
const bytes = bodyBytes(raw);
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors, { kind: "connected-too-large", dataBytes: bytes }); return; }
try { pushBounded(state.connected, JSON.parse(raw)); } catch { pushBounded(state.errors, { kind: "connected-parse" }); }
});
this.addEventListener(config.businessEvent, () => { state.businessEventCount += 1; });
this.addEventListener("workbench.error", (event) => {
try { state.failures.push(JSON.parse(event.data)); } catch { state.errors.push({ kind: "workbench-error-parse" }); }
const raw = String(event.data || "");
const bytes = bodyBytes(raw);
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors, { kind: "workbench-error-too-large", dataBytes: bytes }); return; }
try { pushBounded(state.failures, JSON.parse(raw)); } catch { pushBounded(state.errors, { kind: "workbench-error-parse" }); }
});
this.addEventListener("error", () => { state.errors.push({ kind: "eventsource-error" }); });
this.addEventListener("error", () => { pushBounded(state.errors, { kind: "eventsource-error" }); });
}
}
Object.defineProperty(ObservedEventSource, "CONNECTING", { value: NativeEventSource.CONNECTING });
@@ -685,12 +725,16 @@ async function realtimeArmExistingSessionRefreshProbe(targetPage, profile) {
productEventsPath: profile.productEventsPath,
productReadyEvent: profile.productReadyEvent,
businessEvent: profile.businessEvent,
maxEntries: profile.aggregateBudget.perArrayEntries,
maxEventTextChars: profile.aggregateBudget.perEventBodyBytes,
});
}
async function realtimeWorkbenchDomIdentity() {
return page.evaluate(() => {
const cards = Array.from(document.querySelectorAll("#conversation-list > .message-card"));
return page.evaluate((maxRows) => {
const cardNodes = document.querySelectorAll("#conversation-list > .message-card");
if (cardNodes.length > maxRows) return { overflow: true, messageCount: cardNodes.length, maxRows, identities: [], missingIdentityRows: [], duplicateIdentities: [], traceIds: [], rows: [], valuesRedacted: true };
const cards = Array.from(cardNodes);
const rows = cards.map((element, index) => ({
index,
role: element.getAttribute("data-role") || null,
@@ -709,10 +753,11 @@ async function realtimeWorkbenchDomIdentity() {
rows,
valuesRedacted: true,
};
});
}, maxDomEvidenceRows);
}
function realtimeAssertSameWorkbenchDom(before, after) {
if (before.overflow === true || after.overflow === true) throw new Error("Workbench DOM identity evidence exceeds the YAML row cap");
if (before.missingIdentityRows.length > 0 || after.missingIdentityRows.length > 0) throw new Error("Workbench DOM contains rows without a stable message identity");
if (before.duplicateIdentities.length > 0 || after.duplicateIdentities.length > 0) throw new Error("Workbench DOM contains duplicate message identities");
if (before.messageCount !== after.messageCount) throw new Error("Workbench message count changed across refresh: " + before.messageCount + " -> " + after.messageCount);
@@ -771,6 +816,21 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) {
const terminalTimeoutMs = hasDurationOverride
? Math.min(Number(profile.terminalTimeoutMs), Math.max(1000, Number(durationMs)))
: Number(profile.terminalTimeoutMs);
const entrySlots = 17;
const eventStreamSlots = 8;
const perArrayEntries = Math.max(1, Math.min(maxRealtimeEventEntries, Math.floor(maxRealtimeTotalEntries / entrySlots)));
const bodyRetentionSlots = eventStreamSlots * (perArrayEntries + 1);
const aggregateBudget = {
maxTotalEntries: maxRealtimeTotalEntries,
maxTotalBodyBytes: maxRealtimeTotalBodyBytes,
entrySlots,
eventStreamSlots,
perArrayEntries,
bodyRetentionSlots,
perEventBodyBytes: Math.max(1, Math.min(maxRealtimeEventBodyBytes, Math.floor(maxRealtimeTotalBodyBytes / bodyRetentionSlots))),
allocation: "partitioned-across-network-product-debug-active-arrays",
valuesRedacted: true,
};
return {
...profile,
debugStreams,
@@ -785,6 +845,7 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) {
conditionalEventTypePairs: profile.conditionalEventTypePairs && typeof profile.conditionalEventTypePairs === "object" ? sanitize(profile.conditionalEventTypePairs) : {},
expectedProductSse: sanitize(expectedProductSse),
expectedKafka: sanitize(expectedKafka),
aggregateBudget,
};
}
@@ -795,7 +856,14 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
const key = label + "-" + sessionId;
await subscriberPage.evaluate(({ key: streamKey, sessionId: scopedSessionId, profile: inputProfile }) => {
const root = window.__unideskRealtimeFanoutProducts ??= {};
const state = { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [], source: null };
const state = { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [], source: null, droppedEntries: 0, droppedBodyEntries: 0, droppedBodyBytes: 0 };
const maxEntries = Math.max(1, Number(inputProfile.maxEntries || 1));
const maxEventTextChars = Math.max(1, Number(inputProfile.maxEventTextChars || 1));
const bodyBytes = (value) => new TextEncoder().encode(String(value || "")).byteLength;
const pushBounded = (items, value) => {
items.push(value);
if (items.length > maxEntries) { state.droppedEntries += items.length - maxEntries; items.splice(0, items.length - maxEntries); }
};
root[streamKey] = state;
const firstText = (...values) => {
for (const value of values) if (typeof value === "string" && value.trim()) return value.trim();
@@ -811,12 +879,15 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
state.source = source;
source.onopen = () => { state.openCount += 1; };
const record = (eventName, raw) => {
const textRaw = String(raw || "");
const bytes = bodyBytes(textRaw);
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors, { at: Date.now(), event: "event-too-large", dataBytes: bytes }); return; }
let value;
try { value = JSON.parse(raw); } catch { return; }
try { value = JSON.parse(textRaw); } catch { return; }
const nested = value?.event && typeof value.event === "object" ? value.event : {};
const context = value?.context && typeof value.context === "object" ? value.context : {};
const eventType = firstText(nested.eventType, nested.type, value.eventType, value.type);
state.records.push({
pushBounded(state.records, {
eventName,
at: Date.now(),
schema: firstText(value.schema),
@@ -831,19 +902,21 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
userMessageId: firstText(value.userMessageId, nested.userMessageId, nested.messageId),
terminal: value.terminal === true || nested.terminal === true || eventType === "terminal" || eventType === "terminal_status",
status: firstText(nested.status, value.status),
envelopeFingerprint: fingerprint(raw),
envelopeFingerprint: fingerprint(textRaw),
valuesRedacted: true,
});
if (state.records.length > 800) state.records.splice(0, state.records.length - 800);
};
source.addEventListener(inputProfile.productReadyEvent, (event) => {
state.connectedEventCount += 1;
try { state.connected = JSON.parse(event.data); } catch { state.connected = { parseError: true }; }
const raw = String(event.data || "");
const bytes = bodyBytes(raw);
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; state.connected = { tooLarge: true, dataBytes: bytes }; return; }
try { state.connected = JSON.parse(raw); } catch { state.connected = { parseError: true }; }
});
source.addEventListener(inputProfile.businessEvent, (event) => record(inputProfile.businessEvent, event.data));
source.addEventListener("workbench.error", (event) => state.errors.push({ at: Date.now(), event: "workbench.error", dataBytes: String(event.data || "").length }));
source.onerror = () => state.errors.push({ at: Date.now(), event: "transport.error" });
}, { key, sessionId, profile: { productEventsPath: profile.productEventsPath, productReadyEvent: profile.productReadyEvent, businessEvent: profile.businessEvent } });
source.addEventListener("workbench.error", (event) => pushBounded(state.errors, { at: Date.now(), event: "workbench.error", dataBytes: String(event.data || "").length }));
source.onerror = () => pushBounded(state.errors, { at: Date.now(), event: "transport.error" });
}, { key, sessionId, profile: { productEventsPath: profile.productEventsPath, productReadyEvent: profile.productReadyEvent, businessEvent: profile.businessEvent, maxEntries: profile.aggregateBudget.perArrayEntries, maxEventTextChars: profile.aggregateBudget.perEventBodyBytes } });
return { label, key, context: subscriberContext, page: subscriberPage };
}
@@ -872,7 +945,7 @@ async function realtimeGotoHealth(targetPage, timeoutMs, profile, label) {
wrapped.details = { ...(wrapped.details || {}), label, attempts, valuesRedacted: true };
throw wrapped;
}
await sleep(profile.navigationRetryDelayMs * attempt);
await interruptibleRunnerSleep(profile.navigationRetryDelayMs * attempt);
}
}
throw new Error(label + " health navigation retry exhausted");
@@ -943,8 +1016,8 @@ async function realtimeRunTurn(input) {
if (!admissionOutcome.response) throw new Error("realtime turn admission response failed: " + String(admissionOutcome.error?.message || admissionOutcome.error || "unknown"));
const admission = admissionOutcome.response;
if (admission.status() < 200 || admission.status() >= 300) throw new Error("realtime turn admission returned HTTP " + admission.status());
let admissionBody = {};
try { admissionBody = await admission.json(); } catch {}
const admissionBodyResult = await boundedResponseJson(admission);
const admissionBody = admissionBodyResult.value || {};
const canonicalTraceId = realtimeFirstText(admissionBody?.traceId, admissionBody?.result?.traceId, captured.traceId);
if (canonicalTraceId !== captured.traceId) throw new Error("canonical trace changed after ready barrier");
const nonTerminal = [];
@@ -970,7 +1043,14 @@ async function realtimeRunTurn(input) {
async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
await debug.page.evaluate(({ key: streamKey, traceId: scopedTraceId, profile: inputProfile }) => {
const root = window.__unideskRealtimeFanoutDebug ??= {};
const state = { connected: {}, connectedEventCount: {}, openCount: {}, records: {}, errors: {}, sources: {} };
const state = { connected: {}, connectedEventCount: {}, openCount: {}, records: {}, errors: {}, sources: {}, droppedEntries: 0, droppedBodyEntries: 0, droppedBodyBytes: 0 };
const maxEntries = Math.max(1, Number(inputProfile.maxEntries || 1));
const maxEventTextChars = Math.max(1, Number(inputProfile.maxEventTextChars || 1));
const bodyBytes = (value) => new TextEncoder().encode(String(value || "")).byteLength;
const pushBounded = (items, value) => {
items.push(value);
if (items.length > maxEntries) { state.droppedEntries += items.length - maxEntries; items.splice(0, items.length - maxEntries); }
};
root[streamKey] = state;
const firstText = (...values) => {
for (const value of values) if (typeof value === "string" && value.trim()) return value.trim();
@@ -1001,11 +1081,17 @@ async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
source.onopen = () => { state.openCount[stream] += 1; };
source.addEventListener(inputProfile.debugReadyEvent, (event) => {
state.connectedEventCount[stream] += 1;
try { state.connected[stream] = JSON.parse(event.data); } catch { state.connected[stream] = { parseError: true }; }
const raw = String(event.data || "");
const bytes = bodyBytes(raw);
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; state.connected[stream] = { tooLarge: true, dataBytes: bytes }; return; }
try { state.connected[stream] = JSON.parse(raw); } catch { state.connected[stream] = { parseError: true }; }
});
source.addEventListener("hwlab.kafka.event", (event) => {
const raw = String(event.data || "");
const bytes = bodyBytes(raw);
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors[stream], { at: Date.now(), kind: "event-too-large", dataBytes: bytes }); return; }
let outer;
try { outer = JSON.parse(event.data); } catch { return; }
try { outer = JSON.parse(raw); } catch { return; }
const value = outer?.value && typeof outer.value === "object" ? outer.value : {};
const nested = value?.event && typeof value.event === "object" ? value.event : {};
const payload = nested?.payload && typeof nested.payload === "object" ? nested.payload : {};
@@ -1017,7 +1103,7 @@ async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
: stream === "hwlab"
? firstText(nested.eventType, nested.type, value.eventType)
: firstText(value.eventType, value.stdio?.method);
state.records[stream].push({
pushBounded(state.records[stream], {
stream,
at: Date.now(),
topic: firstText(outer.topic),
@@ -1042,16 +1128,17 @@ async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
envelopeFingerprint: stream === "hwlab" ? fingerprint(JSON.stringify(value)) : null,
valuesRedacted: true,
});
if (state.records[stream].length > 1000) state.records[stream].splice(0, state.records[stream].length - 1000);
});
source.addEventListener("hwlab.kafka.error", (event) => state.errors[stream].push({ at: Date.now(), dataBytes: String(event.data || "").length }));
source.onerror = () => state.errors[stream].push({ at: Date.now(), transport: true });
source.addEventListener("hwlab.kafka.error", (event) => pushBounded(state.errors[stream], { at: Date.now(), dataBytes: String(event.data || "").length }));
source.onerror = () => pushBounded(state.errors[stream], { at: Date.now(), transport: true });
}
}, { key, traceId, profile: {
debugStreams: profile.debugStreams,
debugEventsPath: profile.debugEventsPath,
debugReadyEvent: profile.debugReadyEvent,
fromBeginning: profile.fromBeginning,
maxEntries: profile.aggregateBudget.perArrayEntries,
maxEventTextChars: profile.aggregateBudget.perEventBodyBytes,
} });
}
@@ -1143,7 +1230,7 @@ async function realtimeWaitTurnStable(debug, key, traceId, subscribers, profile)
} else if (Date.now() - stableSince >= profile.terminalQuietMs) {
return { debug: debugSnapshot, products: productSnapshots };
}
await sleep(250);
await interruptibleRunnerSleep(250);
}
throw new Error("realtime debug/product streams did not become quiet after terminal");
}
@@ -1169,7 +1256,7 @@ async function realtimeProductSnapshot(subscriber) {
return subscriber.page.evaluate((key) => {
const state = window.__unideskRealtimeFanoutProducts?.[key];
if (!state) return { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [{ missing: true }] };
return { connected: state.connected ? JSON.parse(JSON.stringify(state.connected)) : null, connectedEventCount: state.connectedEventCount, openCount: state.openCount, records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)) };
return { connected: state.connected ? JSON.parse(JSON.stringify(state.connected)) : null, connectedEventCount: state.connectedEventCount, openCount: state.openCount, records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)), droppedEntries: Number(state.droppedEntries || 0), droppedBodyEntries: Number(state.droppedBodyEntries || 0), droppedBodyBytes: Number(state.droppedBodyBytes || 0) };
}, subscriber.key);
}
@@ -1177,7 +1264,7 @@ async function realtimeDebugSnapshot(debug, key) {
return debug.page.evaluate((streamKey) => {
const state = window.__unideskRealtimeFanoutDebug?.[streamKey];
if (!state) return { connected: {}, connectedEventCount: {}, openCount: {}, records: { stdio: [], agentrun: [], hwlab: [] }, errors: { stdio: [], agentrun: [], hwlab: [] } };
return { connected: JSON.parse(JSON.stringify(state.connected)), connectedEventCount: JSON.parse(JSON.stringify(state.connectedEventCount)), openCount: JSON.parse(JSON.stringify(state.openCount)), records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)) };
return { connected: JSON.parse(JSON.stringify(state.connected)), connectedEventCount: JSON.parse(JSON.stringify(state.connectedEventCount)), openCount: JSON.parse(JSON.stringify(state.openCount)), records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)), droppedEntries: Number(state.droppedEntries || 0), droppedBodyEntries: Number(state.droppedBodyEntries || 0), droppedBodyBytes: Number(state.droppedBodyBytes || 0) };
}, key);
}
@@ -1191,6 +1278,9 @@ async function realtimeAllDebugSnapshots(debug) {
openCount: JSON.parse(JSON.stringify(state.openCount || {})),
records: JSON.parse(JSON.stringify(state.records || {})),
errors: JSON.parse(JSON.stringify(state.errors || {})),
droppedEntries: Number(state.droppedEntries || 0),
droppedBodyEntries: Number(state.droppedBodyEntries || 0),
droppedBodyBytes: Number(state.droppedBodyBytes || 0),
}));
});
}
@@ -1282,7 +1372,7 @@ async function realtimeWaitReplayedTraceJointQuiet(debug, key, subscriber, trace
if (comparison.equal && terminalSeen && sourceTransportStable && quietForMs >= profile.terminalQuietMs) {
return { debug: debugSnapshot, product: productSnapshot, comparison, quietForMs, valuesRedacted: true };
}
await sleep(250);
await interruptibleRunnerSleep(250);
}
const error = new Error(label + " Kafka source and product replay envelope multisets did not converge before the joint quiet timeout");
error.details = {
@@ -1589,6 +1679,8 @@ function realtimeTurnSummary(turn, debug, product, terminal) {
}
async function realtimeCaptureSubscriberScreenshot(targetPage, reportDir, name, summary) {
const screenshotSkipped = reserveScreenshot("realtime-fanout-" + name);
if (screenshotSkipped !== null) return screenshotSkipped;
await targetPage.evaluate((value) => {
document.documentElement.style.background = "#0b1020";
document.body.innerHTML = "";
@@ -1598,7 +1690,7 @@ async function realtimeCaptureSubscriberScreenshot(targetPage, reportDir, name,
document.body.appendChild(pre);
}, summary);
const file = path.join(reportDir, name);
await targetPage.screenshot({ path: file, fullPage: true, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
await targetPage.screenshot({ path: file, fullPage: false, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
const meta = await fileMeta(file);
const artifact = { kind: "realtime-fanout-screenshot", path: file, name, byteCount: meta.byteCount, sha256: meta.sha256, commandId: activeCommandId, valuesRedacted: true };
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...artifact });
@@ -1679,7 +1771,7 @@ async function realtimePoll(read, timeoutMs, label) {
while (Date.now() < deadline) {
const value = await read();
if (value) return value;
await sleep(250);
await interruptibleRunnerSleep(250);
}
throw new Error(label + " timed out after " + timeoutMs + "ms");
}
@@ -57,11 +57,50 @@ async function writeManifest(extra = {}) {
navigation: { maxAttempts: navigationMaxAttempts, valuesRedacted: true },
pageAuthority: { browser: "chromium", context: "shared-auth", pageMode: "dual-control-observer", controlPageId: pageId, observerPageId, continuityBreaksRecorded: true },
pageProvenance: compactPageProvenance(currentPageProvenance),
sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false },
sampling: {
mode: "passive",
sampleIntervalMs,
screenshotIntervalMs,
maxSamples,
maxRunMs,
maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0,
observerRefreshIntervalMs,
observerInitiatedDefault: false,
responseBodyReadDefault: false,
},
resourceLimits: {
maxScreenshots,
maxPerformanceEntries,
maxBrowserProcessHistoryEntries,
maxBrowserFreezeSignalHistoryEntries,
maxRealtimeEventEntries,
maxRealtimeEventBodyBytes,
maxDomEvidenceRows,
maxProcessScanEntries,
maxPerformanceCaptureMs,
responseBodyMaxBytes,
webPerformanceRequestBodyMaxBytes,
maxJsonlQueueEntries,
maxJsonlQueueBytes,
maxJsonlLineBytes,
jsonlFlushTimeoutMs,
maxResponseBodyInFlight,
maxPassiveListenerTasks,
passiveTaskFlushTimeoutMs,
closeStepTimeoutMs,
maxProcessReadConcurrency,
maxRealtimeTotalEntries,
maxRealtimeTotalBodyBytes,
valuesRedacted: true,
},
alertThresholds,
browserFreezePolicy,
projectManagement,
jsonlRotation,
jsonlWriter: jsonlWriterStats(),
responseBodies: responseBodyStats(),
passiveTasks: passiveTaskStats(),
realtimeAggregate: realtimeAggregateLast,
commandDirs: dirs,
artifacts: files,
safety: { pureClient: true, inboundApi: false, database: false, queueConsumer: false, k8sWorkload: false, valuesRedacted: true, secretValuesPrinted: false },
@@ -84,11 +123,47 @@ async function writeHeartbeat(extra = {}) {
currentUrl: currentPageUrl(),
observerUrl: pageUrl(observerPage),
observerRefreshIntervalMs,
observerLifecycle: {
maxSamples,
maxRunMs,
maxRunSeconds: maxRunMs > 0 ? Math.ceil(maxRunMs / 1000) : 0,
valuesRedacted: true,
},
resourceLimits: {
screenshotCount,
maxScreenshots,
maxPerformanceEntries,
maxBrowserProcessHistoryEntries,
maxBrowserFreezeSignalHistoryEntries,
maxRealtimeEventEntries,
maxRealtimeEventBodyBytes,
maxDomEvidenceRows,
maxProcessScanEntries,
maxPerformanceCaptureMs,
responseBodyMaxBytes,
webPerformanceRequestBodyMaxBytes,
maxJsonlQueueEntries,
maxJsonlQueueBytes,
maxJsonlLineBytes,
jsonlFlushTimeoutMs,
maxResponseBodyInFlight,
maxPassiveListenerTasks,
passiveTaskFlushTimeoutMs,
closeStepTimeoutMs,
maxProcessReadConcurrency,
maxRealtimeTotalEntries,
maxRealtimeTotalBodyBytes,
valuesRedacted: true,
},
lastObserverRefreshAt: Number.isFinite(lastObserverRefreshAtMs) ? new Date(lastObserverRefreshAtMs).toISOString() : null,
pageProvenance: compactPageProvenance(currentPageProvenance),
sampleSeq,
commandSeq,
activeCommandId,
jsonlWriter: jsonlWriterStats(),
responseBodies: responseBodyStats(),
passiveTasks: passiveTaskStats(),
realtimeAggregate: realtimeAggregateLast,
updatedAt: new Date().toISOString(),
uptimeMs: Date.now() - startedAtMs,
...extra,
@@ -96,6 +171,22 @@ async function writeHeartbeat(extra = {}) {
await writeFile(files.heartbeat, JSON.stringify(heartbeat, null, 2) + "\n", { mode: 0o600 });
}
async function writeTerminalResourceStats(extra = {}) {
for (const file of [files.manifest, files.heartbeat]) {
let existing = {};
try { existing = JSON.parse(await readFile(file, "utf8")); } catch {}
await writeFile(file, JSON.stringify({
...existing,
jsonlWriter: jsonlWriterStats(),
responseBodies: responseBodyStats(),
passiveTasks: passiveTaskStats(),
terminalResources: sanitize(extra),
updatedAt: new Date().toISOString(),
valuesRedacted: true,
}, null, 2) + "\n", { mode: 0o600 });
}
}
function startHeartbeatPulse() {
const timer = setInterval(() => {
if (stopping) return;
@@ -319,6 +410,7 @@ function recordBrowserFreezeSignal(kind, sample, pageMetric, detail) {
const windowMs = browserFreezePolicy.blockerWindowMs;
const cutoff = signal.tsMs - windowMs;
while (browserFreezeSignalHistory.length > 0 && Number(browserFreezeSignalHistory[0].tsMs || 0) < cutoff) browserFreezeSignalHistory.shift();
while (browserFreezeSignalHistory.length > maxBrowserFreezeSignalHistoryEntries) browserFreezeSignalHistory.shift();
return {
kind,
rootCause: detail.rootCause,
@@ -374,62 +466,96 @@ async function triggerBrowserFreezeBlocker(trigger) {
}
async function terminateBrowserForFreezeBlocker(blocker) {
const childProcess = browser && typeof browser.process === "function" ? browser.process() : null;
const pid = Number(childProcess?.pid);
if (!Number.isFinite(pid) || pid <= 0) {
return { ok: false, reason: "browser-process-unavailable", blockerKind: blocker.kind, valuesRedacted: true };
const result = await terminateTrackedBrowserProcesses("freeze-blocker:" + String(blocker.kind || "unknown"));
return { ...result, blockerKind: blocker.kind, valuesRedacted: true };
}
function rememberTrackedBrowserProcesses(processes) {
const merged = new Map(trackedBrowserProcessIdentities.map((item) => [item.pid + ":" + item.startTicks, item]));
for (const item of Array.isArray(processes) ? processes : []) {
if (!Number.isInteger(item.pid) || !Number.isInteger(item.startTicks) || !item.role) continue;
merged.set(item.pid + ":" + item.startTicks, {
pid: item.pid,
ppid: item.ppid,
pgid: item.pgid,
sid: item.sid,
startTicks: item.startTicks,
role: item.role,
commandHash: item.commandHash || null,
valuesRedacted: true,
});
}
trackedBrowserProcessIdentities = [...merged.values()].slice(-Math.min(maxProcessScanEntries, 1024));
}
async function matchingTrackedBrowserProcesses() {
const rows = await Promise.all(trackedBrowserProcessIdentities.map(async (identity) => {
const current = await readOneProcProcessStat(identity.pid).catch(() => null);
return current && current.startTicks === identity.startTicks ? { ...identity, currentPpid: current.ppid, currentPgid: current.pgid, currentSid: current.sid } : null;
}));
return rows.filter(Boolean);
}
async function waitForTrackedBrowserExit(timeoutMs, pollIntervalMs) {
const deadline = Date.now() + timeoutMs;
let remaining = await matchingTrackedBrowserProcesses();
while (remaining.length > 0 && Date.now() <= deadline) {
await boundedCleanupSleep(pollIntervalMs);
remaining = await matchingTrackedBrowserProcesses();
}
return remaining;
}
async function terminateTrackedBrowserProcesses(reason, requiredDiscovery = null) {
let refreshDiscovery;
try {
const summary = await collectChromiumProcessSummary(null);
refreshDiscovery = { ok: true, processScan: summary.processScan, chromiumProcessCount: summary.chromiumProcessCount, valuesRedacted: true };
} catch (error) {
refreshDiscovery = { ok: false, processScan: lastProcessScan, error: errorSummary(error), valuesRedacted: true };
}
const discovery = requiredDiscovery || refreshDiscovery;
const identityScanUnavailable = browserLaunchAttempted === true
&& (discovery?.ok !== true || discovery?.processScan?.complete !== true);
const initial = await matchingTrackedBrowserProcesses();
const result = {
ok: false,
pid: Math.floor(pid),
ok: initial.length === 0 && !identityScanUnavailable,
reason,
browserLaunchAttempted,
identityScanUnavailable,
requiredDiscovery: discovery,
refreshDiscovery,
trackedIdentityCount: trackedBrowserProcessIdentities.length,
initialLiveCount: initial.length,
gracefulSignal: browserFreezePolicy.kill.gracefulSignal,
forceSignal: browserFreezePolicy.kill.forceSignal,
graceMs: browserFreezePolicy.kill.graceMs,
pollIntervalMs: browserFreezePolicy.kill.pollIntervalMs,
gracefulSent: false,
forceSent: false,
exitedAfterGrace: false,
exitedAfterForce: false,
gracefulSentCount: 0,
forceSentCount: 0,
remainingIdentityCount: initial.length,
valuesRedacted: true,
};
try {
childProcess.kill(browserFreezePolicy.kill.gracefulSignal);
result.gracefulSent = true;
} catch (error) {
result.gracefulError = errorSummary(error);
}
result.exitedAfterGrace = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
if (!result.exitedAfterGrace) {
for (const item of [...initial].sort((a, b) => b.pid - a.pid)) {
try {
childProcess.kill(browserFreezePolicy.kill.forceSignal);
result.forceSent = true;
} catch (error) {
result.forceError = errorSummary(error);
}
result.exitedAfterForce = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
process.kill(item.pid, browserFreezePolicy.kill.gracefulSignal);
result.gracefulSentCount += 1;
} catch {}
}
result.ok = result.exitedAfterGrace || result.exitedAfterForce;
let remaining = await waitForTrackedBrowserExit(browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
for (const item of [...remaining].sort((a, b) => b.pid - a.pid)) {
try {
process.kill(item.pid, browserFreezePolicy.kill.forceSignal);
result.forceSentCount += 1;
} catch {}
}
remaining = await waitForTrackedBrowserExit(browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
result.remainingIdentityCount = remaining.length;
result.remainingIdentities = remaining.slice(0, 50).map((item) => ({ pid: item.pid, startTicks: item.startTicks, role: item.role, originalPpid: item.ppid, currentPpid: item.currentPpid, originalSid: item.sid, currentSid: item.currentSid, valuesRedacted: true }));
result.ok = remaining.length === 0 && !identityScanUnavailable;
return result;
}
async function waitForPidExit(pid, timeoutMs, pollIntervalMs) {
const deadline = Date.now() + timeoutMs;
while (Date.now() <= deadline) {
if (!pidAlive(pid)) return true;
await sleep(pollIntervalMs);
}
return !pidAlive(pid);
}
function pidAlive(pid) {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
function browserProcessSampleRef(sample) {
return {
ts: sample?.ts ?? null,
@@ -496,13 +622,16 @@ async function collectChromiumProcessSummary(browserPid) {
queue.push(child);
}
}
const processes = all
.filter((item) => descendants.has(item.pid))
const detailed = await readProcProcessDetails(all.filter((item) => descendants.has(item.pid)));
const processes = detailed
.map((item) => ({ ...item, role: classifyChromiumProcess(item) }))
.filter((item) => item.role)
.map((item) => ({
pid: item.pid,
ppid: item.ppid,
pgid: item.pgid,
sid: item.sid,
startTicks: item.startTicks,
name: item.name || null,
role: item.role,
rssBytes: item.rssBytes,
@@ -512,6 +641,7 @@ async function collectChromiumProcessSummary(browserPid) {
valuesRedacted: true,
}))
.sort((a, b) => Number(b.rssBytes || 0) - Number(a.rssBytes || 0));
rememberTrackedBrowserProcesses(processes);
const roles = {};
for (const item of processes) {
const role = item.role || "unknown";
@@ -525,6 +655,7 @@ async function collectChromiumProcessSummary(browserPid) {
const maxProcess = processes[0] || null;
return {
sampledRootPids: roots,
processScan: lastProcessScan,
chromiumProcessCount: processes.length,
totalRssBytes,
totalRssMb: bytesToMb(totalRssBytes),
@@ -541,28 +672,68 @@ async function readProcProcessTable() {
let entries = [];
try {
entries = await readdir("/proc");
} catch {
return [];
} catch (error) {
lastProcessScan = { ok: false, complete: false, totalPidEntries: 0, scannedPidEntries: 0, failedPidEntries: 0, truncated: false, readConcurrency: maxProcessReadConcurrency, error: errorSummary(error), valuesRedacted: true };
throw new Error("proc-process-table-unavailable");
}
const numeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0);
const rows = await Promise.all(numeric.map((pid) => readOneProcProcess(pid).catch(() => null)));
return rows.filter(Boolean);
const allNumeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0).sort((a, b) => b - a);
const numeric = allNumeric.slice(0, maxProcessScanEntries);
const rows = [];
for (let offset = 0; offset < numeric.length; offset += maxProcessReadConcurrency) {
const batch = await Promise.all(numeric.slice(offset, offset + maxProcessReadConcurrency).map((pid) => readOneProcProcessStat(pid).catch(() => null)));
rows.push(...batch.filter(Boolean));
}
const failedPidEntries = numeric.length - rows.length;
const truncated = numeric.length < allNumeric.length;
const unusable = numeric.length > 0 && rows.length === 0;
lastProcessScan = {
ok: !unusable,
complete: !unusable && !truncated,
totalPidEntries: allNumeric.length,
scannedPidEntries: numeric.length,
readablePidEntries: rows.length,
failedPidEntries,
truncated,
readConcurrency: maxProcessReadConcurrency,
error: unusable ? "all-proc-stat-reads-failed" : null,
valuesRedacted: true,
};
if (unusable) throw new Error("proc-process-table-unreadable");
return rows;
}
async function readOneProcProcess(pid) {
const [statText, statusText, cmdlineText] = await Promise.all([
readFile(path.join("/proc", String(pid), "stat"), "utf8").catch(() => ""),
async function readOneProcProcessStat(pid) {
const statText = await readFile(path.join("/proc", String(pid), "stat"), "utf8");
if (!statText) return null;
return { pid, ...procIdentityFromStat(statText) };
}
async function readProcProcessDetails(rows) {
const detailed = [];
for (let offset = 0; offset < rows.length; offset += maxProcessReadConcurrency) {
const batch = await Promise.all(rows.slice(offset, offset + maxProcessReadConcurrency).map((row) => readOneProcProcessDetails(row).catch(() => null)));
detailed.push(...batch.filter(Boolean));
}
return detailed;
}
async function readOneProcProcessDetails(row) {
const pid = row.pid;
const [statusText, cmdlineText] = await Promise.all([
readFile(path.join("/proc", String(pid), "status"), "utf8").catch(() => ""),
readFile(path.join("/proc", String(pid), "cmdline"), "utf8").catch(() => ""),
]);
if (!statText && !statusText) return null;
const ppid = procPpidFromStat(statText);
if (!statusText && !cmdlineText) return null;
const ppid = row.ppid;
const name = procStatusField(statusText, "Name") || null;
const rssKb = procStatusKb(statusText, "VmRSS");
const vmSizeKb = procStatusKb(statusText, "VmSize");
return {
pid,
ppid,
pgid: row.pgid,
sid: row.sid,
startTicks: row.startTicks,
name,
cmdline: String(cmdlineText || "").replace(/\0/gu, " ").replace(/\s+/gu, " ").trim(),
rssBytes: Number.isFinite(rssKb) ? rssKb * 1024 : 0,
@@ -570,13 +741,13 @@ async function readOneProcProcess(pid) {
};
}
function procPpidFromStat(value) {
function procIdentityFromStat(value) {
const text = String(value || "");
const end = text.lastIndexOf(")");
if (end < 0) return null;
if (end < 0) return { ppid: null, pgid: null, sid: null, startTicks: null };
const parts = text.slice(end + 1).trim().split(/\s+/u);
const ppid = Number(parts[1]);
return Number.isFinite(ppid) ? ppid : null;
const integerAt = (index) => Number.isInteger(Number(parts[index])) ? Number(parts[index]) : null;
return { ppid: integerAt(1), pgid: integerAt(2), sid: integerAt(3), startTicks: integerAt(19) };
}
function procStatusField(value, key) {
@@ -623,6 +794,7 @@ function updateBrowserProcessHistory(tsMs, processSummary) {
browserProcessHistory.push({ tsMs, totalRssBytes, maxProcessRssBytes });
const retentionMs = Math.max(windowMs * 3, 180000);
while (browserProcessHistory.length > 0 && tsMs - browserProcessHistory[0].tsMs > retentionMs) browserProcessHistory.shift();
while (browserProcessHistory.length > maxBrowserProcessHistoryEntries) browserProcessHistory.shift();
const windowStartMs = tsMs - windowMs;
const candidates = browserProcessHistory.filter((item) => item.tsMs >= windowStartMs && item.tsMs <= tsMs);
const baseline = candidates[0] || browserProcessHistory[0] || null;
@@ -688,7 +860,13 @@ function applyBrowserPageRuntimeBaseline(result) {
const key = [result.pageRole || "unknown", result.pageId || "unknown", Number.isFinite(Number(result.pageEpoch)) ? Number(result.pageEpoch) : 0].join(":");
const current = browserPageRuntimeMemorySnapshot(result);
const existing = browserPageRuntimeBaselines.get(key);
if (!existing && current) browserPageRuntimeBaselines.set(key, { ...current, capturedAt: new Date().toISOString(), source: "first-page-runtime-sample", valuesRedacted: true });
if (!existing && current) {
const prefix = [result.pageRole || "unknown", result.pageId || "unknown"].join(":") + ":";
for (const existingKey of browserPageRuntimeBaselines.keys()) {
if (existingKey.startsWith(prefix)) browserPageRuntimeBaselines.delete(existingKey);
}
browserPageRuntimeBaselines.set(key, { ...current, capturedAt: new Date().toISOString(), source: "first-page-runtime-sample", valuesRedacted: true });
}
const baseline = browserPageRuntimeBaselines.get(key) || null;
result.baseline = baseline ? {
capturedAt: baseline.capturedAt,
@@ -810,8 +988,10 @@ function bytesToMb(value) {
}
function attachPassiveListeners(targetPage, pageRole = "control", targetPageId = pageId) {
void installPagePerformanceProbe(targetPage, pageRole, targetPageId)
.catch((error) => appendJsonl(files.errors, eventRecord("performance-probe-install-error", { pageRole, pageId: targetPageId, error: errorSummary(error), valuesRedacted: true })));
trackPassiveTask(
() => installPagePerformanceProbe(targetPage, pageRole, targetPageId),
"performance-probe-install:" + pageRole,
);
targetPage.on("request", (request) => {
const webPerformancePayload = summarizeWebPerformanceRequestPayload(request);
void appendJsonl(files.network, eventRecord("request", {
@@ -841,19 +1021,10 @@ function attachPassiveListeners(targetPage, pageRole = "control", targetPageId =
statusText: response.statusText(),
fromServiceWorker: response.fromServiceWorker(),
};
void (async () => {
trackPassiveTask(async () => {
const bodyFields = await summarizeWorkbenchResponseBody(response, request);
await appendJsonl(files.network, eventRecord("response", { ...base, ...bodyFields }));
})().catch((error) => appendJsonl(files.errors, eventRecord("response-body-summary-error", {
pageRole,
pageId: targetPageId,
sampleSeq,
commandId: activeCommandId,
method: request.method(),
url: safeUrl(response.url()),
error: errorSummary(error),
valuesRedacted: true
})));
}, "response-body-summary:" + pageRole);
});
targetPage.on("requestfailed", (request) => {
void appendJsonl(files.network, eventRecord("requestfailed", {
@@ -795,8 +795,32 @@ function digestSessionRail(value) {
};
}
function reserveScreenshot(reason) {
if (screenshotCount >= maxScreenshots) {
lastScreenshotAtMs = Date.now();
return {
seq: null,
sampleSeq,
ts: new Date().toISOString(),
kind: "screenshot-skipped",
reason,
skipped: true,
skipReason: "screenshot-cap-reached",
screenshotCount,
maxScreenshots,
pageId,
currentUrl: currentPageUrl(),
valuesRedacted: true,
};
}
screenshotCount += 1;
return null;
}
async function captureScreenshot(reason, imageType = "png") {
if (!page || page.isClosed()) throw new Error("page is not available for screenshot");
const skipped = reserveScreenshot(reason);
if (skipped !== null) return skipped;
if (screenshotCaptureState && screenshotCaptureState.settled !== true) {
const ageMs = Date.now() - Number(screenshotCaptureState.startedAtMs || Date.now());
const error = new Error("screenshot capture already in progress");
@@ -887,10 +911,10 @@ async function summarizeWorkbenchResponseBody(response, request) {
const headers = response.headers();
const contentType = String(headers["content-type"] || headers["Content-Type"] || "");
if (!/json/iu.test(contentType)) return { bodyRead: false, bodyReadSkipped: "non-json", valuesRedacted: true };
const contentLength = Number(headers["content-length"] || headers["Content-Length"]);
const maxBytes = 512 * 1024;
if (Number.isFinite(contentLength) && contentLength > maxBytes) return { bodyRead: false, bodyReadSkipped: "content-length-too-large", bodyByteCount: contentLength, valuesRedacted: true };
const text = await response.text();
const maxBytes = responseBodyMaxBytes;
const body = await boundedResponseText(response, maxBytes);
if (body.ok !== true || typeof body.value !== "string") return { bodyRead: false, bodyReadSkipped: body.error?.reason || "bounded-response-read-failed", bodyContract: body.bodyContract, valuesRedacted: true };
const text = body.value;
const byteCount = Buffer.byteLength(text);
if (byteCount > maxBytes) return { bodyRead: true, bodyReadSkipped: "body-too-large", bodyByteCount: byteCount, bodyHash: sha256Text(text), valuesRedacted: true };
let parsed = null;
@@ -15,6 +15,7 @@ import { nodeWebObserveRunnerTraceReadabilitySource } from "./hwlab-node-web-obs
export function nodeWebObserveRunnerSource(): string {
return String.raw`#!/usr/bin/env node
import { createHash, randomBytes } from "node:crypto";
import { createReadStream } from "node:fs";
import { appendFile, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
@@ -31,9 +32,30 @@ const targetPath = process.env.UNIDESK_WEB_OBSERVE_TARGET_PATH || "/workbench";
const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS, 5000);
const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000);
const screenshotCaptureTimeoutMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_CAPTURE_TIMEOUT_MS, 15000, 1000, 120000);
const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0);
const maxSamples = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, "UNIDESK_WEB_OBSERVE_MAX_SAMPLES", 1, 10000000);
const observerRefreshIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS, 180000);
const maxRunMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, 0);
const maxRunMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, "UNIDESK_WEB_OBSERVE_MAX_RUN_MS", 1000, 86400000);
const maxScreenshots = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS, "UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS", 1, 10000);
const maxPerformanceEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES", 1, 100000);
const maxBrowserProcessHistoryEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES", 1, 100000);
const maxBrowserFreezeSignalHistoryEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES", 1, 100000);
const maxRealtimeEventEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES", 1, 100000);
const maxRealtimeEventBodyBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES", 1, 16777216);
const maxDomEvidenceRows = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS, "UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS", 1, 100000);
const maxProcessScanEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES", 1, 1000000);
const maxPerformanceCaptureMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS, "UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS", 1, 600) * 1000;
const responseBodyMaxBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES, "UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES", 1024, 16777216);
const maxJsonlQueueEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES", 1, 100000);
const maxJsonlQueueBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES, "UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES", 1024, 67108864);
const maxJsonlLineBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES, "UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES", 1024, 16777216);
const jsonlFlushTimeoutMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS, "UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS", 100, 60000);
const maxResponseBodyInFlight = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT, "UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT", 1, 64);
const maxPassiveListenerTasks = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS, "UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS", 1, 10000);
const passiveTaskFlushTimeoutMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS, "UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS", 100, 60000);
const closeStepTimeoutMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS, "UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS", 100, 60000);
const maxProcessReadConcurrency = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY, "UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY", 1, 256);
const maxRealtimeTotalEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES", 1, 100000);
const maxRealtimeTotalBodyBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES", 1024, 67108864);
const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900");
const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE || "auto");
const origin = {
@@ -44,7 +66,7 @@ const origin = {
browserProxyModeSource: process.env.UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE_SOURCE || "default",
valuesRedacted: true,
};
const webPerformanceRequestBodyMaxBytes = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES, 65536, 1024, 1048576);
const webPerformanceRequestBodyMaxBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES, "UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES", 1024, 1048576);
const authLoginMaxAttempts = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_ATTEMPTS, 6, 1, 20);
const authLoginRequestTimeoutMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_REQUEST_TIMEOUT_MS, 30000, 1000, 120000);
const authLoginInitialDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_INITIAL_DELAY_MS, 500, 0, 60000);
@@ -92,6 +114,7 @@ let lastObserverRefreshAtMs = Date.now();
let sampleSeq = 0;
let commandSeq = 0;
let artifactSeq = 0;
let screenshotCount = 0;
let activeCommandId = null;
let activeCommandType = null;
let stopping = false;
@@ -107,10 +130,30 @@ let heartbeatPulseTimer = null;
let browserProcessMonitorStop = null;
let browserProcessMonitorSeq = 0;
let browserFreezeBlocker = null;
let shutdownSignal = null;
let browserResourcesClosePromise = null;
let browserLaunchAttempted = false;
const browserProcessHistory = [];
const browserPageRuntimeBaselines = new Map();
const browserFreezeSignalHistory = [];
let trackedBrowserProcessIdentities = [];
const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] };
const jsonlWriterState = { queue: [], queuedBytes: 0, inFlight: 0, pump: null, acceptedEntries: 0, acceptedBytes: 0, writtenEntries: 0, writtenBytes: 0, droppedEntries: 0, droppedBytes: 0, failedEntries: 0, dropsByFile: {} };
const responseBodyState = { inFlight: 0, accepted: 0, skippedSaturated: 0, completed: 0, failed: 0 };
const passiveTaskState = { tasks: new Set(), accepted: 0, dropped: 0, completed: 0, failed: 0, drainTimedOut: false };
const runnerSleepWakes = new Set();
let lastProcessScan = { ok: false, complete: false, totalPidEntries: 0, scannedPidEntries: 0, failedPidEntries: 0, truncated: false, readConcurrency: maxProcessReadConcurrency, error: "not-scanned", valuesRedacted: true };
let realtimeAggregateLast = null;
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
process.once(signal, () => {
shutdownSignal = signal;
stopping = true;
terminalStatus = "stopping";
interruptRunnerSleep();
void closeBrowserResources("signal:" + signal);
});
}
try {
if (!password) throw new Error("missing HWLAB_WEB_PASS");
@@ -122,6 +165,7 @@ try {
if (jsonlRotation.files.length > 0) await appendJsonl(files.control, eventRecord("jsonl-rotated", { stamp: jsonlRotation.stamp, archiveDir: path.relative(stateDir, dirs.archive), files: jsonlRotation.files, valuesRedacted: true }));
const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
const { chromium } = await launcher.importPlaywright();
browserLaunchAttempted = true;
browser = await launcher.launchChromium(chromium, chromiumLaunchOptions);
context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) });
page = await context.newPage();
@@ -152,19 +196,19 @@ try {
await writeManifest({ status: "running", auth: publicAuth(auth) });
await writeHeartbeat({ status: "running" });
while (!stopping) {
await drainOneCommand();
if (browserFreezeBlocker) break;
await samplePage("interval");
if (browserFreezeBlocker) break;
if (maxSamples > 0 && sampleSeq >= maxSamples) {
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
break;
}
if (maxRunMs > 0 && Date.now() - startedAtMs >= maxRunMs) {
await appendJsonl(files.control, controlRecord({ id: "max-run-ms", type: "stop", source: "sampler" }, "completed", { reason: "max-run-ms", maxRunMs, elapsedMs: Date.now() - startedAtMs }));
break;
}
await sleep(sampleIntervalMs);
if (maxSamples > 0 && sampleSeq >= maxSamples) {
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
break;
}
await drainOneCommand();
if (browserFreezeBlocker) break;
await samplePage("interval");
if (browserFreezeBlocker) break;
await interruptibleRunnerSleep(sampleIntervalMs);
}
if (browserFreezeBlocker) {
terminalStatus = "failed";
@@ -186,7 +230,49 @@ try {
} finally {
if (browserProcessMonitorStop) browserProcessMonitorStop();
if (heartbeatPulseTimer) clearInterval(heartbeatPulseTimer);
if (browser) await browser.close().catch(() => {});
interruptRunnerSleep();
const closeResult = await closeBrowserResources(shutdownSignal === null ? "finally" : "signal:" + shutdownSignal);
const passiveTaskDrain = await drainPassiveTasks();
const terminalFlush = await flushJsonlQueue();
await writeTerminalResourceStats({ closeResult, passiveTaskDrain, terminalFlush }).catch(() => {});
}
async function closeBrowserResources(reason) {
if (browserResourcesClosePromise) return browserResourcesClosePromise;
browserResourcesClosePromise = (async () => {
stopping = true;
const closingObserverPage = observerPage;
const closingPage = page;
const closingContext = context;
const closingBrowser = browser;
let preCloseIdentityDiscovery;
try {
const summary = await collectChromiumProcessSummary(null);
preCloseIdentityDiscovery = { ok: true, processScan: summary.processScan, chromiumProcessCount: summary.chromiumProcessCount, valuesRedacted: true };
} catch (error) {
preCloseIdentityDiscovery = { ok: false, processScan: lastProcessScan, error: errorSummary(error), valuesRedacted: true };
}
observerPage = null;
page = null;
context = null;
browser = null;
const steps = [];
const closeStep = async (label, operation) => {
try {
await withHardTimeout(Promise.resolve().then(operation), closeStepTimeoutMs, label + " close exceeded " + closeStepTimeoutMs + "ms");
steps.push({ label, ok: true, valuesRedacted: true });
} catch (error) {
steps.push({ label, ok: false, error: errorSummary(error), valuesRedacted: true });
}
};
if (closingObserverPage && !closingObserverPage.isClosed()) await closeStep("observer-page", () => closingObserverPage.close());
if (closingPage && !closingPage.isClosed()) await closeStep("control-page", () => closingPage.close());
if (closingContext) await closeStep("browser-context", () => closingContext.close());
if (closingBrowser) await closeStep("browser", () => closingBrowser.close());
const browserCleanup = await terminateTrackedBrowserProcesses("browser-close-residual", preCloseIdentityDiscovery);
return { ok: steps.every((step) => step.ok === true) && browserCleanup.ok === true, reason, closeStepTimeoutMs, steps, browserCleanup, valuesRedacted: true };
})();
return browserResourcesClosePromise;
}
${nodeWebObserveRunnerRuntimeSource()}
@@ -300,7 +300,7 @@ async function workbenchTraceReadabilityWaitForCurrentProductCard(timeoutMs, pol
if (stableCardCount === 1 && timelineCount === 1) return { card, traceId, scope: lastScope };
}
}
await sleep(pollIntervalMs);
await interruptibleRunnerSleep(pollIntervalMs);
}
const error = new Error("在 owning YAML 指定的 " + timeoutMs + "ms 内未找到当前新运行中的产品 Trace 卡片");
error.details = { scope: lastScope, valuesRedacted: true };
@@ -313,7 +313,7 @@ async function workbenchTraceReadabilityWaitForDisclosureOpen(card, traceId, tim
while (Date.now() <= deadline) {
latest = await workbenchTraceReadabilitySnapshot(card, traceId);
if (latest.disclosure.open === true) return latest;
await sleep(pollIntervalMs);
await interruptibleRunnerSleep(pollIntervalMs);
}
return latest;
}
@@ -422,7 +422,7 @@ async function workbenchTraceReadabilityWaitForReadableRunning(card, traceId, ti
if (workbenchTraceReadabilityTerminalStatus(latest.status)) {
return { observed: false, reason: "terminal-before-readable-running-sample", elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
}
await sleep(pollIntervalMs);
await interruptibleRunnerSleep(pollIntervalMs);
}
return { observed: false, reason: "readable-running-timeout", elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
}
@@ -436,7 +436,7 @@ async function workbenchTraceReadabilityWaitForTerminal(card, traceId, timeoutMs
if (workbenchTraceReadabilityTerminalStatus(latest.status)) {
return { observed: true, status: latest.status, elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
}
await sleep(pollIntervalMs);
await interruptibleRunnerSleep(pollIntervalMs);
}
return { observed: false, status: latest.status, elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
}
@@ -466,7 +466,7 @@ async function workbenchTraceReadabilityWaitForStableTerminal(card, traceId, qui
stableSinceMs = null;
}
previousSignature = signature;
await sleep(pollIntervalMs);
await interruptibleRunnerSleep(pollIntervalMs);
}
return { observed: workbenchTraceReadabilityTerminalStatus(latest.status), stable: false, sampleCount, elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
}
@@ -2,13 +2,221 @@
// Responsibility: Runner low-level IO, URL, YAML-derived policy parsing, redaction, and timeout utility source fragment.
export function nodeWebObserveRunnerUtilitySource(): string {
return String.raw`async function appendJsonl(file, value) {
await appendFile(file, JSON.stringify(sanitize(value)) + "\n", { mode: 0o600 });
return String.raw`function appendJsonl(file, value) {
let line;
try {
line = JSON.stringify(sanitize(value)) + "\n";
} catch {
recordJsonlDrop(file, 0);
return Promise.resolve({ ok: false, dropped: true, reason: "json-encode-failed", valuesRedacted: true });
}
const bytes = Buffer.byteLength(line);
if (bytes > maxJsonlLineBytes) {
recordJsonlDrop(file, bytes);
return Promise.resolve({ ok: false, dropped: true, reason: "entry-too-large", bytes, maxJsonlLineBytes, valuesRedacted: true });
}
if (jsonlWriterState.queue.length >= maxJsonlQueueEntries || jsonlWriterState.queuedBytes + bytes > maxJsonlQueueBytes) {
recordJsonlDrop(file, bytes);
return Promise.resolve({ ok: false, dropped: true, reason: "queue-saturated", bytes, valuesRedacted: true });
}
jsonlWriterState.acceptedEntries += 1;
jsonlWriterState.acceptedBytes += bytes;
jsonlWriterState.queuedBytes += bytes;
const completion = new Promise((resolve) => jsonlWriterState.queue.push({ file, line, bytes, resolve }));
startJsonlPump();
return completion;
}
function recordJsonlDrop(file, bytes) {
jsonlWriterState.droppedEntries += 1;
jsonlWriterState.droppedBytes += Math.max(0, Number(bytes) || 0);
const key = path.basename(String(file || "unknown")).slice(0, 80);
jsonlWriterState.dropsByFile[key] = (jsonlWriterState.dropsByFile[key] || 0) + 1;
}
function startJsonlPump() {
if (jsonlWriterState.pump) return jsonlWriterState.pump;
jsonlWriterState.pump = (async () => {
while (jsonlWriterState.queue.length > 0) {
const entry = jsonlWriterState.queue.shift();
jsonlWriterState.queuedBytes = Math.max(0, jsonlWriterState.queuedBytes - entry.bytes);
jsonlWriterState.inFlight = 1;
try {
await appendFile(entry.file, entry.line, { mode: 0o600 });
jsonlWriterState.writtenEntries += 1;
jsonlWriterState.writtenBytes += entry.bytes;
entry.resolve({ ok: true, dropped: false, valuesRedacted: true });
} catch (error) {
jsonlWriterState.failedEntries += 1;
entry.resolve({ ok: false, dropped: false, error: errorSummary(error), valuesRedacted: true });
} finally {
jsonlWriterState.inFlight = 0;
}
}
})().finally(() => {
jsonlWriterState.pump = null;
if (jsonlWriterState.queue.length > 0) startJsonlPump();
});
return jsonlWriterState.pump;
}
function jsonlWriterStats() {
return {
queueEntries: jsonlWriterState.queue.length,
queueBytes: jsonlWriterState.queuedBytes,
inFlight: jsonlWriterState.inFlight,
maxQueueEntries: maxJsonlQueueEntries,
maxQueueBytes: maxJsonlQueueBytes,
maxLineBytes: maxJsonlLineBytes,
acceptedEntries: jsonlWriterState.acceptedEntries,
acceptedBytes: jsonlWriterState.acceptedBytes,
writtenEntries: jsonlWriterState.writtenEntries,
writtenBytes: jsonlWriterState.writtenBytes,
droppedEntries: jsonlWriterState.droppedEntries,
droppedBytes: jsonlWriterState.droppedBytes,
failedEntries: jsonlWriterState.failedEntries,
dropsByFile: { ...jsonlWriterState.dropsByFile },
valuesRedacted: true,
};
}
async function flushJsonlQueue() {
try {
await withHardTimeout((async () => {
while (jsonlWriterState.queue.length > 0 || jsonlWriterState.inFlight > 0 || jsonlWriterState.pump) {
if (jsonlWriterState.pump) await jsonlWriterState.pump;
else if (jsonlWriterState.queue.length > 0) await startJsonlPump();
else break;
}
})(), jsonlFlushTimeoutMs, "jsonl terminal flush exceeded " + jsonlFlushTimeoutMs + "ms");
return { ok: true, timedOut: false, stats: jsonlWriterStats(), valuesRedacted: true };
} catch (error) {
return { ok: false, timedOut: true, error: errorSummary(error), stats: jsonlWriterStats(), valuesRedacted: true };
}
}
async function fileMeta(file) {
const [buffer, stats] = await Promise.all([readFile(file), stat(file)]);
return { byteCount: stats.size, sha256: "sha256:" + createHash("sha256").update(buffer).digest("hex") };
const digestPromise = new Promise((resolve, reject) => {
const digest = createHash("sha256");
const stream = createReadStream(file);
stream.on("data", (chunk) => digest.update(chunk));
stream.once("error", reject);
stream.once("end", () => resolve("sha256:" + digest.digest("hex")));
});
const [sha256, stats] = await Promise.all([digestPromise, stat(file)]);
return { byteCount: stats.size, sha256 };
}
function responseBodyReadContract(response, maxBytes = responseBodyMaxBytes) {
let headers = {};
try { headers = response?.headers?.() || {}; } catch {}
const contentEncoding = String(headers["content-encoding"] || headers["Content-Encoding"] || "").trim().toLowerCase();
if (contentEncoding && contentEncoding !== "identity") {
return { ok: false, reason: "encoded-body-size-unbounded", contentEncoding: truncate(contentEncoding, 80), contentLength: null, maxBytes, valuesRedacted: true };
}
const rawContentLength = headers["content-length"] || headers["Content-Length"];
const contentLength = Number(rawContentLength);
if (!Number.isFinite(contentLength) || contentLength < 0) {
return { ok: false, reason: "content-length-unavailable", contentLength: null, maxBytes, valuesRedacted: true };
}
if (contentLength > maxBytes) {
return { ok: false, reason: "content-length-too-large", contentLength, maxBytes, valuesRedacted: true };
}
return { ok: true, reason: null, contentLength, maxBytes, valuesRedacted: true };
}
async function boundedResponseJson(response, maxBytes = responseBodyMaxBytes) {
const bodyContract = responseBodyReadContract(response, maxBytes);
if (bodyContract.ok !== true) return { ok: false, value: null, error: bodyContract, bodyContract, valuesRedacted: true };
const permit = tryAcquireResponseBodyPermit();
if (!permit) return { ok: false, value: null, error: { reason: "response-body-inflight-saturated", valuesRedacted: true }, bodyContract, valuesRedacted: true };
try {
const value = await response.json();
responseBodyState.completed += 1;
return { ok: true, value, error: null, bodyContract, valuesRedacted: true };
} catch (error) {
responseBodyState.failed += 1;
return { ok: false, value: null, error: errorSummary(error), bodyContract, valuesRedacted: true };
} finally {
permit.release();
}
}
async function boundedResponseText(response, maxBytes = responseBodyMaxBytes) {
const bodyContract = responseBodyReadContract(response, maxBytes);
if (bodyContract.ok !== true) return { ok: false, value: null, error: bodyContract, bodyContract, valuesRedacted: true };
const permit = tryAcquireResponseBodyPermit();
if (!permit) return { ok: false, value: null, error: { reason: "response-body-inflight-saturated", valuesRedacted: true }, bodyContract, valuesRedacted: true };
try {
const value = await response.text();
responseBodyState.completed += 1;
return { ok: true, value, error: null, bodyContract, valuesRedacted: true };
} catch (error) {
responseBodyState.failed += 1;
return { ok: false, value: null, error: errorSummary(error), bodyContract, valuesRedacted: true };
} finally {
permit.release();
}
}
function tryAcquireResponseBodyPermit() {
if (responseBodyState.inFlight >= maxResponseBodyInFlight) {
responseBodyState.skippedSaturated += 1;
return null;
}
responseBodyState.inFlight += 1;
responseBodyState.accepted += 1;
let released = false;
return { release() { if (released) return; released = true; responseBodyState.inFlight = Math.max(0, responseBodyState.inFlight - 1); } };
}
function responseBodyStats() {
return { ...responseBodyState, maxInFlight: maxResponseBodyInFlight, valuesRedacted: true };
}
function trackPassiveTask(factory, label) {
if (stopping || passiveTaskState.tasks.size >= maxPassiveListenerTasks) {
passiveTaskState.dropped += 1;
return false;
}
passiveTaskState.accepted += 1;
let task;
task = Promise.resolve()
.then(factory)
.then(() => { passiveTaskState.completed += 1; })
.catch(async (error) => {
passiveTaskState.failed += 1;
await appendJsonl(files.errors, eventRecord("passive-task-error", { label, error: errorSummary(error), valuesRedacted: true })).catch(() => {});
})
.finally(() => passiveTaskState.tasks.delete(task));
passiveTaskState.tasks.add(task);
task.catch(() => {});
return true;
}
function passiveTaskStats() {
return {
inFlight: passiveTaskState.tasks.size,
maxInFlight: maxPassiveListenerTasks,
accepted: passiveTaskState.accepted,
dropped: passiveTaskState.dropped,
completed: passiveTaskState.completed,
failed: passiveTaskState.failed,
drainTimedOut: passiveTaskState.drainTimedOut,
valuesRedacted: true,
};
}
async function drainPassiveTasks() {
try {
await withHardTimeout((async () => {
while (passiveTaskState.tasks.size > 0) await Promise.allSettled([...passiveTaskState.tasks]);
})(), passiveTaskFlushTimeoutMs, "passive task drain exceeded " + passiveTaskFlushTimeoutMs + "ms");
return { ok: true, timedOut: false, stats: passiveTaskStats(), valuesRedacted: true };
} catch (error) {
passiveTaskState.drainTimedOut = true;
return { ok: false, timedOut: true, error: errorSummary(error), stats: passiveTaskStats(), valuesRedacted: true };
}
}
function currentPageUrl() {
@@ -77,6 +285,14 @@ function boundedInteger(value, fallback, min, max) {
return integer;
}
function requiredBoundedInteger(value, name, min, max) {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < min || parsed > max) {
throw new Error(name + " must come from the owning YAML and be an integer between " + min + " and " + max);
}
return parsed;
}
function positiveNumber(value, fallback) {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
@@ -306,9 +522,33 @@ function errorSummary(error) {
}
function sleep(ms) {
return interruptibleRunnerSleep(ms);
}
function boundedCleanupSleep(ms) {
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
}
function interruptibleRunnerSleep(ms) {
if (stopping) return Promise.resolve();
return new Promise((resolve) => {
const timer = setTimeout(() => {
runnerSleepWakes.delete(wake);
resolve();
}, Math.max(0, ms));
const wake = () => {
clearTimeout(timer);
runnerSleepWakes.delete(wake);
resolve();
};
runnerSleepWakes.add(wake);
});
}
function interruptRunnerSleep() {
for (const wake of [...runnerSleepWakes]) wake();
}
function withHardTimeout(promise, timeoutMs, message) {
const guarded = Promise.resolve(promise);
let timer = null;
@@ -1285,12 +1285,10 @@ async function launchWorkbenchFromTask(command) {
}, { timeout: 30000 }).then(async (response) => {
let chatPayload = null;
let chatPayloadError = null;
try {
chatPayload = await response.json();
} catch (error) {
chatPayloadError = errorSummary(error);
}
const headers = response.headers();
const chatBody = await boundedResponseJson(response);
chatPayload = chatBody.value;
chatPayloadError = chatBody.error;
return {
observed: true,
status: response.status(),
@@ -1316,11 +1314,9 @@ async function launchWorkbenchFromTask(command) {
const launchTraceHeader = typeof headers["x-hwlab-otel-trace-id"] === "string" ? headers["x-hwlab-otel-trace-id"] : null;
let payload = null;
let responseParseError = null;
try {
payload = await launchResponse.json();
} catch (error) {
responseParseError = errorSummary(error);
}
const launchBody = await boundedResponseJson(launchResponse);
payload = launchBody.value;
responseParseError = launchBody.error;
const sessionId = sessionIdFromAgentSessionPayload(payload);
const workbenchUrl = safeUrlPath(payload?.workbenchUrl || payload?.url || "");
const contractVersion = typeof payload?.contractVersion === "string" ? payload.contractVersion : null;
@@ -0,0 +1,261 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { test } from "bun:test";
import type { CommandResult } from "./command";
import { rootPath } from "./config";
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeMemoryStartGuardConfigRef, webProbeScreenshotFullPage } from "./hwlab-node-web-probe-memory-guard";
import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source";
const spec = hwlabRuntimeLaneSpecForNode("v03", "NC01");
const thresholdBytes = 4 * 1024 * 1024 * 1024;
function memoryPayload(availableBytes: number, extra: Record<string, unknown> = {}): Record<string, unknown> {
return {
ok: true,
action: "gc remote memory",
providerId: "NC01",
metric: "MemAvailable",
source: "/proc/meminfo",
swapExcluded: true,
memory: { availableBytes, availableHuman: "fixture", ...extra },
valuesRedacted: true,
};
}
function exportedFunctionSource(source: string, name: string): string {
const start = source.indexOf(`export function ${name}(`);
assert.ok(start >= 0, `missing exported function ${name}`);
const nextExport = source.indexOf("\nexport function ", start + 1);
const nextFunction = source.indexOf("\nfunction ", start + 1);
const next = [nextExport, nextFunction].filter((value) => value >= 0).sort((left, right) => left - right)[0] ?? -1;
return source.slice(start, next < 0 ? source.length : next);
}
test("WebProbe memory threshold is read from the single owning YAML policy", () => {
const policy = spec.webProbe?.resourcePolicy?.memoryStartGuard;
assert.equal(policy?.thresholdBytes, thresholdBytes);
assert.equal(policy?.metric, "MemAvailable");
assert.equal(policy?.source, "/proc/meminfo");
assert.equal(policy?.swapExcluded, true);
assert.equal(policy?.manualComparator, "less-than-or-equal");
assert.equal(policy?.sentinelComparator, "less-than");
assert.match(webProbeMemoryStartGuardConfigRef, /resourcePolicy\.memoryStartGuard$/u);
});
test("browser evidence limits and viewport screenshots are read from owning YAML", () => {
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
assert.deepEqual(policy, {
screenshotMode: "viewport",
maxNetworkRequestEntries: 200,
maxNetworkRequestBytes: 262144,
maxNetworkFailureEntries: 100,
maxNetworkFailureBytes: 131072,
});
const assignments = webProbeBrowserEvidenceEnvAssignments(spec).join(" ");
assert.match(assignments, /UNIDESK_WEB_PROBE_SCREENSHOT_MODE='viewport'/u);
assert.match(assignments, /UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES='200'/u);
assert.match(assignments, /UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES='131072'/u);
});
test("manual start is blocked at exactly 4 GiB and requires controlled GC then MemAvailable recheck", () => {
const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", memoryPayload(thresholdBytes));
const memory = guard.memory as Record<string, unknown>;
const gcNext = guard.gcNext as Record<string, unknown>;
assert.equal(guard.status, "blocked");
assert.equal(guard.startBlocked, true);
assert.equal(guard.decision, "gc-required");
assert.equal(memory.metric, "MemAvailable");
assert.equal(memory.source, "/proc/meminfo");
assert.equal(memory.swapExcluded, true);
assert.equal(memory.comparator, "less-than-or-equal");
assert.equal(memory.thresholdMatched, true);
assert.equal(gcNext.requiredBeforeStart, true);
assert.match(String(gcNext.planCommand), /gc remote NC01 plan --memory-pressure-only$/u);
assert.match(String(gcNext.runCommand), /gc remote NC01 run --confirm --memory-pressure-only$/u);
assert.match(String(gcNext.statusCommandTemplate), /gc remote NC01 status --job-id <job-id>$/u);
assert.match(String(gcNext.recheckCommand), /gc remote NC01 memory$/u);
assert.equal(gcNext.recheckMetric, "MemAvailable");
assert.deepEqual(gcNext.requiredOrder, ["plan", "run", "status-until-terminal", "memory-recheck"]);
assert.deepEqual(gcNext.startRemainsBlockedUntil, ["gc-job-terminal", "MemAvailable-threshold-cleared"]);
});
test("sentinel allows exactly 4 GiB but skips one byte below without considering swap", () => {
const exact = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes, { swapFreeBytes: 12 * 1024 ** 3 }));
const below = evaluateWebProbeMemoryGuard(spec, "sentinel-cadence", memoryPayload(thresholdBytes - 1, { swapFreeBytes: 12 * 1024 ** 3 }));
const belowMemory = below.memory as Record<string, unknown>;
assert.equal(exact.status, "allowed");
assert.equal(exact.startBlocked, false);
assert.equal((exact.memory as Record<string, unknown>).comparator, "less-than");
assert.equal(below.status, "skipped");
assert.equal(below.startBlocked, true);
assert.equal(below.decision, "skip-observer");
assert.equal(belowMemory.availableBytes, thresholdBytes - 1);
assert.equal(belowMemory.swapExcluded, true);
});
test("memory guard rejects a snapshot that is not the /proc/meminfo MemAvailable contract", () => {
const guard = evaluateWebProbeMemoryGuard(spec, "manual-start", {
...memoryPayload(thresholdBytes + 1),
source: "cgroup-v2-memory.current",
});
assert.equal(guard.status, "blocked");
assert.equal(guard.reason, "memory-snapshot-contract-invalid");
assert.equal(guard.startBlocked, true);
});
test("memory guard executes only the controlled read-only remote memory entry", () => {
let observedCommand: string[] = [];
let observedTimeoutMs = 0;
const execute = (command: string[], cwd: string, options: { timeoutMs?: number }): CommandResult => {
observedCommand = command;
observedTimeoutMs = options.timeoutMs ?? 0;
return {
command,
cwd,
exitCode: 0,
stdout: JSON.stringify({ ok: true, command: "gc remote NC01 memory", data: memoryPayload(thresholdBytes + 1) }),
stderr: "",
signal: null,
timedOut: false,
};
};
const guard = runWebProbeMemoryGuard(spec, "manual-start", execute);
assert.deepEqual(observedCommand, ["bun", "scripts/cli.ts", "gc", "remote", "NC01", "memory"]);
assert.equal(observedTimeoutMs, 30_000);
assert.equal(guard.status, "allowed");
});
test("remote memory collector reads MemAvailable directly and excludes swap", () => {
const source = readFileSync(rootPath("scripts/src/gc-remote-runner.py"), "utf8");
const start = source.indexOf("def collect_memory_snapshot():");
const end = source.indexOf("\ndef observe_run_record", start);
const collector = source.slice(start, end);
assert.match(collector, /source = "\/proc\/meminfo"/u);
assert.match(collector, /fields\.get\("MemAvailable"\)/u);
assert.match(collector, /"swapExcluded": True/u);
assert.doesNotMatch(collector, /SwapFree|SwapTotal|\["free", "-b"\]/u);
});
test("manual and sentinel gates return before any observer launch path", () => {
const manualSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe-actions.ts"), "utf8");
const manualStart = manualSource.indexOf("export function runNodeWebProbeObserveStart");
const manualGuard = manualSource.indexOf("runWebProbeMemoryGuard(spec, options.memoryGuardMode)", manualStart);
const manualReturn = manualSource.indexOf("startBlocked: true", manualGuard);
const manualJob = manualSource.indexOf("const jobId =", manualStart);
const manualLaunch = manualSource.indexOf("setsid env", manualStart);
assert.ok(manualStart >= 0 && manualGuard > manualStart && manualReturn > manualGuard);
assert.ok(manualReturn < manualJob && manualJob < manualLaunch);
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5-observe.ts"), "utf8");
const quickVerify = sentinelSource.indexOf("export function runSentinelQuickVerify");
const sentinelGuard = sentinelSource.indexOf("runWebProbeMemoryGuard(state.spec, \"sentinel-cadence\")", quickVerify);
const sentinelSkip = sentinelSource.indexOf("return recordSkippedQuickVerify", sentinelGuard);
const providerPreflight = sentinelSource.indexOf("runSentinelExactProviderProfilePreflight", quickVerify);
const observeStart = sentinelSource.indexOf('"web-probe", "observe", "start"', quickVerify);
assert.ok(quickVerify >= 0 && sentinelGuard > quickVerify && sentinelSkip > sentinelGuard);
assert.ok(sentinelSkip < providerPreflight && providerPreflight < observeStart);
assert.match(sentinelSource.slice(observeStart, observeStart + 800), /--sentinel-cadence/u);
const serviceSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-service.ts"), "utf8");
const plannedStart = serviceSource.indexOf('"bun", "scripts/cli.ts", "web-probe", "observe", "start"');
assert.match(serviceSource.slice(plannedStart, plannedStart + 800), /--sentinel-cadence/u);
});
test("all browser-launch entries use the shared preflight and final launch guard", () => {
const probeSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
const scriptSource = readFileSync(rootPath("scripts/src/hwlab-node/web-observe-scripts.ts"), "utf8");
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8");
const parserSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8");
const run = exportedFunctionSource(probeSource, "runNodeWebProbe");
const screenshot = exportedFunctionSource(probeSource, "runNodeWebProbeScreenshot");
const asyncRun = exportedFunctionSource(probeSource, "runNodeWebProbeAsync");
const script = exportedFunctionSource(scriptSource, "runNodeWebProbeScript");
const scriptShell = exportedFunctionSource(scriptSource, "nodeWebProbeScriptRemoteShell");
const dashboard = exportedFunctionSource(sentinelSource, "probeSentinelDashboardBrowser");
const parser = exportedFunctionSource(parserSource, "parseNodeWebProbeOptions");
assert.doesNotMatch(probeSource, /runNodeWebProbeMemoryGuard/u);
assert.match(run, /if \(options\.action === "script"\) return runNodeWebProbeScript/u);
assert.match(run, /if \(options\.commandTimeoutSeconds > 55\) return runNodeWebProbeAsync/u);
assert.match(run, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start"/u);
assert.match(screenshot, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start", webProbeScreenshotRemoteScript/u);
assert.match(asyncRun, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+webProbeGuardedLaunchShell\(spec, "manual-start"[\s\S]+"structured-output"\)/u);
assert.match(script, /runWebProbeMemoryGuard\(spec, "manual-start"\)[\s\S]+nodeWebProbeScriptRemoteShell/u);
assert.match(scriptShell, /webProbeGuardedLaunchShell\(spec, "manual-start"/u);
assert.match(parser, /if \(actionRaw === "opencode-smoke"\)[\s\S]+action: "script"/u);
assert.match(dashboard, /runWebProbeMemoryGuard\(state\.spec, "sentinel-cadence"\)[\s\S]+webProbeGuardedLaunchShell\(state\.spec, "sentinel-cadence"/u);
assert.match(dashboard, /UNIDESK_SENTINEL_DASHBOARD_TRIGGER=.*options\.action === "trigger"/u);
assert.match(dashboard, /UNIDESK_SENTINEL_DASHBOARD_CAPTURE=.*options\.action === "screenshot"/u);
});
test("read-only and control-plane entries stay outside browser launch guard", () => {
const dispatchSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
const dispatchStart = dispatchSource.indexOf("export function runNodeWebProbe(");
const dispatchEnd = dispatchSource.indexOf("export function runNodeWebProbeScreenshot", dispatchStart);
const dispatch = dispatchSource.slice(dispatchStart, dispatchEnd);
const readOnlyReturn = dispatch.indexOf('options.action === "observe" && options.observeAction !== "start"');
const firstGuard = dispatch.indexOf("runWebProbeMemoryGuard");
assert.ok(readOnlyReturn >= 0 && firstGuard > readOnlyReturn);
assert.match(dispatch.slice(0, readOnlyReturn), /options\.action === "sentinel"/u);
assert.match(dispatch.slice(0, readOnlyReturn), /options\.action === "screenshot"/u);
const sentinelDispatch = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-cicd.ts"), "utf8");
const sentinel = exportedFunctionSource(sentinelDispatch, "runWebProbeSentinelCommand");
assert.doesNotMatch(sentinel, /runWebProbeMemoryGuard|webProbeGuardedLaunchShell/u);
assert.match(sentinel, /options\.kind === "dashboard"/u);
});
test("generated runner bounds network evidence and defaults screenshots to viewport", () => {
const source = nodeWebProbeScriptRunnerSource();
assert.match(source, /maxNetworkRequestEntries/u);
assert.match(source, /maxNetworkRequestBytes/u);
assert.match(source, /maxNetworkFailureEntries/u);
assert.match(source, /maxNetworkFailureBytes/u);
assert.match(source, /apiRequestEvidenceDropped/u);
assert.match(source, /apiFailureEvidenceDropped/u);
assert.doesNotMatch(source, /fullPage: true/u);
assert.equal((source.match(/fullPage: screenshotMode !== "viewport"/gu) ?? []).length, 2);
const parserSource = exportedFunctionSource(readFileSync(rootPath("scripts/src/hwlab-node/web-probe.ts"), "utf8"), "parseNodeWebProbeOptions");
assert.match(parserSource, /fullPage: args\.includes\("--full-page"\) && !args\.includes\("--no-full-page"\)/u);
const screenshotSource = readFileSync(rootPath("scripts/src/hwlab-node/web-probe-observe.ts"), "utf8");
assert.match(screenshotSource, /webProbeScreenshotFullPage\(spec, options\.fullPage\)/u);
assert.match(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE === "1"/u);
assert.doesNotMatch(screenshotSource, /UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE !== "0"/u);
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5.ts"), "utf8");
assert.match(sentinelSource, /webProbeScreenshotFullPage\(state\.spec, options\.fullPage\)/u);
assert.match(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE === "1"/u);
assert.doesNotMatch(sentinelSource, /UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0"/u);
assert.equal(webProbeScreenshotFullPage(spec, false), false);
assert.equal(webProbeScreenshotFullPage({ ...spec, webProbe: { ...spec.webProbe!, resourcePolicy: { ...spec.webProbe!.resourcePolicy!, browserEvidence: { ...spec.webProbe!.resourcePolicy!.browserEvidence!, screenshotMode: "full-page" } } } }, false), true);
assert.equal(webProbeScreenshotFullPage(spec, true), true);
});
test("final launch guard rechecks MemAvailable under the configured lock", () => {
const sync = webProbeGuardedLaunchShell(spec, "manual-start", "node runner.mjs");
const async = webProbeGuardedLaunchShell(spec, "manual-start", "node helper.mjs start", "structured-output");
assert.match(sync, /flock -w '15'/u);
assert.match(sync, /MemAvailable:/u);
assert.match(sync, /mem_available_bytes.*-le '4294967296'/u);
assert.match(sync, /setsid sh -c/u);
assert.match(sync, /exec 9>&-/u);
assert.equal(sync.includes("playwright"), false);
assert.equal(sync.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true);
assert.equal(sync.includes("trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group"), true);
assert.equal(sync.includes('kill -TERM -- "-$browser_job_pid"'), true);
assert.equal(sync.includes('kill -0 -- "-$browser_job_pid"'), true);
assert.equal(sync.includes('kill -KILL -- "-$browser_job_pid"'), true);
assert.equal(async.includes("browserCreated === true"), false);
assert.equal(async.includes("data.workerPid, data.jobPid, data.pid"), true);
assert.equal(async.includes("[data.browserPid, data.workerPid"), false);
assert.equal(async.includes("reportedBrowserPid === null || row.pid === reportedBrowserPid"), true);
assert.equal(async.includes("then flock -u 9; exit 43; fi"), true);
assert.equal(async.includes("fs.readlinkSync('/proc/' + name + '/exe')"), true);
});
@@ -0,0 +1,324 @@
// SPEC: PJ2026-01060508 Web 哨兵。WebProbe 启动前的 YAML-first 物理内存资格判定。
import type { CommandResult } from "./command";
import { runCommand } from "./command";
import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery";
import { repoRoot } from "./config";
import type { HwlabRuntimeLaneSpec, HwlabRuntimeWebProbeMemoryStartGuardSpec } from "./hwlab-node-lanes";
export const webProbeMemoryStartGuardConfigRef = "config/hwlab-node-lanes.yaml#templates.hwlabV03.webProbeWorkbench.resourcePolicy.memoryStartGuard";
export type WebProbeMemoryGuardMode = "manual-start" | "sentinel-cadence";
export type WebProbeMemoryGuardStatus = "allowed" | "blocked" | "skipped";
type MemoryGuardExecutor = (
command: string[],
cwd: string,
options: { timeoutMs?: number },
) => CommandResult;
function record(value: unknown): Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
function memoryPolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeMemoryStartGuardSpec | null {
return spec.webProbe?.resourcePolicy?.memoryStartGuard ?? null;
}
function shellQuote(value: string): string {
return `'${value.replace(/'/g, `'"'"'`)}'`;
}
export function webProbeBrowserEvidenceEnvAssignments(spec: HwlabRuntimeLaneSpec): string[] {
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML");
return [
`UNIDESK_WEB_PROBE_SCREENSHOT_MODE=${shellQuote(policy.screenshotMode)}`,
`UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES=${shellQuote(String(policy.maxNetworkRequestEntries))}`,
`UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES=${shellQuote(String(policy.maxNetworkRequestBytes))}`,
`UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES=${shellQuote(String(policy.maxNetworkFailureEntries))}`,
`UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES=${shellQuote(String(policy.maxNetworkFailureBytes))}`,
];
}
export function webProbeScreenshotFullPage(spec: HwlabRuntimeLaneSpec, explicitFullPage: boolean): boolean {
const policy = spec.webProbe?.resourcePolicy?.browserEvidence;
if (policy === undefined) throw new Error("web-probe browser evidence policy is missing from the owning YAML");
return explicitFullPage || policy.screenshotMode !== "viewport";
}
export function webProbeGuardedLaunchShell(
spec: HwlabRuntimeLaneSpec,
mode: WebProbeMemoryGuardMode,
command: string,
readiness: "child-process" | "structured-output" = "child-process",
): string {
const policy = memoryPolicy(spec);
if (policy === null) throw new Error("web-probe memory start policy is missing from the owning YAML");
const closeStepTimeoutMs = spec.webProbe?.resourcePolicy?.observerLifecycle.closeStepTimeoutMs;
if (closeStepTimeoutMs === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML");
const cleanupPollMilliseconds = policy.launchReadyPollMilliseconds;
const cleanupPollSeconds = Math.max(0.05, cleanupPollMilliseconds / 1000);
const cleanupPollAttempts = Math.max(1, Math.ceil(closeStepTimeoutMs / cleanupPollMilliseconds));
const comparator = mode === "manual-start" ? "-le" : "-lt";
const common = [
`exec 9>${shellQuote(policy.launchLockPath)}`,
`flock -w ${shellQuote(String(policy.launchLockTimeoutSeconds))} 9`,
"mem_available_kib=$(awk '$1 == \"MemAvailable:\" { print $2; exit }' /proc/meminfo)",
"case \"$mem_available_kib\" in ''|*[!0-9]*) printf '%s\\n' 'web-probe final memory guard could not read MemAvailable' >&2; exit 42;; esac",
"mem_available_bytes=$((mem_available_kib * 1024))",
`if [ "$mem_available_bytes" ${comparator} ${shellQuote(String(policy.thresholdBytes))} ]; then printf '%s\\n' 'web-probe final memory guard blocked browser launch' >&2; exit 42; fi`,
];
if (readiness === "structured-output") return [
...common,
"launch_output=$(mktemp)",
"launch_error=$(mktemp)",
"trap 'rm -f \"$launch_output\" \"$launch_error\"' EXIT",
`if ! (exec 9>&-; ${command}) >"$launch_output" 2>"$launch_error"; then cat "$launch_output"; cat "$launch_error" >&2; exit 43; fi`,
"cat \"$launch_output\"",
"cat \"$launch_error\" >&2",
`if ! node - "$launch_output" ${shellQuote(String(policy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(policy.launchReadyPollMilliseconds))} <<'UNIDESK_WEB_PROBE_STRUCTURED_READINESS'`,
"const fs = require('fs');",
"const text = fs.readFileSync(process.argv[2], 'utf8').trim();",
"const timeoutMs = Number(process.argv[3]);",
"const pollMs = Number(process.argv[4]);",
"let value = null; try { value = JSON.parse(text); } catch {}",
"const data = value && typeof value === 'object' && value.data && typeof value.data === 'object' ? value.data : value;",
"if (!data || typeof data !== 'object' || !data.jobId) process.exit(43);",
"const rootPid = [data.workerPid, data.jobPid, data.pid].find((item) => Number.isInteger(item) && item > 0);",
"if (!rootPid) process.exit(43);",
"const reportedBrowserPid = Number.isInteger(data.browserPid) && data.browserPid > 0 ? data.browserPid : null;",
"const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
"const browserDescendant = () => {",
" const rows = [];",
" for (const name of fs.readdirSync('/proc')) {",
" if (!/^[0-9]+$/.test(name)) continue;",
" try {",
" const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');",
" const close = stat.lastIndexOf(')');",
" const fields = stat.slice(close + 2).split(' ');",
" const ppid = Number(fields[1]);",
" const executable = fs.readlinkSync('/proc/' + name + '/exe');",
" rows.push({ pid: Number(name), ppid, executable });",
" } catch {}",
" }",
" const descendants = new Set([rootPid]);",
" let changed = true;",
" while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
" return rows.some((row) => descendants.has(row.pid) && (reportedBrowserPid === null || row.pid === reportedBrowserPid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable));",
"};",
"(async () => {",
" const deadline = Date.now() + timeoutMs;",
" while (Date.now() <= deadline) { if (browserDescendant()) process.exit(0); await sleep(pollMs); }",
" process.exit(43);",
"})().catch(() => process.exit(43));",
"UNIDESK_WEB_PROBE_STRUCTURED_READINESS",
"then flock -u 9; exit 43; fi",
"flock -u 9",
].join("\n");
return [
...common,
`setsid sh -c ${shellQuote(`exec 9>&-\n${command}`)} &`,
"browser_job_pid=$!",
"cleanup_browser_group() {",
" kill -TERM -- \"-$browser_job_pid\" 2>/dev/null || true",
" cleanup_attempt=0",
` while kill -0 -- "-$browser_job_pid" 2>/dev/null && [ "$cleanup_attempt" -lt ${cleanupPollAttempts} ]; do cleanup_attempt=$((cleanup_attempt + 1)); sleep ${cleanupPollSeconds}; done`,
" if kill -0 -- \"-$browser_job_pid\" 2>/dev/null; then kill -KILL -- \"-$browser_job_pid\" 2>/dev/null || true; fi",
" wait \"$browser_job_pid\" 2>/dev/null || true",
"}",
"trap 'browser_exit=$?; trap - EXIT TERM INT HUP; cleanup_browser_group; exit \"$browser_exit\"' EXIT",
"trap 'exit 143' TERM",
"trap 'exit 130' INT",
"trap 'exit 129' HUP",
"browser_ready=false",
`browser_ready_deadline=$(( $(date +%s) + ${policy.launchReadyTimeoutSeconds} ))`,
"while kill -0 \"$browser_job_pid\" 2>/dev/null; do",
" if node - \"$browser_job_pid\" <<'UNIDESK_WEB_PROBE_CHILD_READINESS'",
"const fs = require('fs');",
"const rootPid = Number(process.argv[2]);",
"const rows = [];",
"for (const name of fs.readdirSync('/proc')) {",
" if (!/^[0-9]+$/.test(name)) continue;",
" try {",
" const stat = fs.readFileSync('/proc/' + name + '/stat', 'utf8');",
" const close = stat.lastIndexOf(')');",
" const fields = stat.slice(close + 2).split(' ');",
" rows.push({ pid: Number(name), ppid: Number(fields[1]), executable: fs.readlinkSync('/proc/' + name + '/exe') });",
" } catch {}",
"}",
"const descendants = new Set([rootPid]);",
"let changed = true;",
"while (changed) { changed = false; for (const row of rows) if (descendants.has(row.ppid) && !descendants.has(row.pid)) { descendants.add(row.pid); changed = true; } }",
"process.exit(rows.some((row) => descendants.has(row.pid) && /(^|\\/)(chromium|chrome|google-chrome)(-[^/]*)?$/i.test(row.executable)) ? 0 : 1);",
"UNIDESK_WEB_PROBE_CHILD_READINESS",
" then browser_ready=true; break; fi",
" if [ \"$(date +%s)\" -ge \"$browser_ready_deadline\" ]; then break; fi",
` sleep ${Math.max(0.05, policy.launchReadyPollMilliseconds / 1000)}`,
"done",
"flock -u 9",
"if [ \"$browser_ready\" != true ]; then",
" printf '%s\\n' 'web-probe browser launch readiness was not confirmed' >&2",
" exit 43",
"fi",
"set +e",
"wait \"$browser_job_pid\"",
"browser_exit=$?",
"set -e",
"exit \"$browser_exit\"",
].join("\n");
}
function humanBytes(value: number | null): string | null {
if (value === null || !Number.isFinite(value)) return null;
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
let current = value;
let index = 0;
while (Math.abs(current) >= 1024 && index < units.length - 1) {
current /= 1024;
index += 1;
}
return `${current.toFixed(index === 0 ? 0 : 2)} ${units[index]}`;
}
function controlledGcNext(node: string): Record<string, unknown> {
return {
automaticGc: false,
requiredOrder: ["plan", "run", "status-until-terminal", "memory-recheck"],
scope: "memory-pressure-only",
planCommand: `bun scripts/cli.ts gc remote ${node} plan --memory-pressure-only`,
runCommand: `bun scripts/cli.ts gc remote ${node} run --confirm --memory-pressure-only`,
jobIdSource: "gc remote run result.data.jobId",
statusCommandTemplate: `bun scripts/cli.ts gc remote ${node} status --job-id <job-id>`,
terminalStatuses: ["succeeded", "failed", "completed", "blocked"],
statusRequiredWhenJobIdReturned: true,
recheckCommand: `bun scripts/cli.ts gc remote ${node} memory`,
recheckMetric: "MemAvailable",
recheckSource: "/proc/meminfo",
swapExcluded: true,
startRemainsBlockedUntil: ["gc-job-terminal", "MemAvailable-threshold-cleared"],
valuesRedacted: true,
};
}
function unavailableGuard(spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, reason: string): Record<string, unknown> {
const policy = memoryPolicy(spec);
const comparator = policy === null
? null
: mode === "manual-start"
? policy.manualComparator
: policy.sentinelComparator;
return {
ok: false,
status: "blocked" satisfies WebProbeMemoryGuardStatus,
eligible: false,
startBlocked: true,
decision: "do-not-start-observer",
reason,
mode,
policyConfigRef: webProbeMemoryStartGuardConfigRef,
memory: {
metric: policy?.metric ?? "MemAvailable",
source: policy?.source ?? "/proc/meminfo",
swapExcluded: policy?.swapExcluded ?? true,
availableBytes: null,
availableHuman: null,
thresholdBytes: policy?.thresholdBytes ?? null,
thresholdHuman: humanBytes(policy?.thresholdBytes ?? null),
comparator,
thresholdMatched: null,
valuesRedacted: true,
},
gcNext: controlledGcNext(spec.nodeId),
valuesRedacted: true,
};
}
export function evaluateWebProbeMemoryGuard(
spec: HwlabRuntimeLaneSpec,
mode: WebProbeMemoryGuardMode,
snapshotPayload: unknown,
): Record<string, unknown> {
const policy = memoryPolicy(spec);
if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
const payload = record(snapshotPayload);
const memory = record(payload.memory);
const availableBytes = typeof memory.availableBytes === "number" && Number.isSafeInteger(memory.availableBytes) && memory.availableBytes >= 0
? memory.availableBytes
: null;
const metadataMatches = payload.ok === true
&& payload.action === "gc remote memory"
&& payload.metric === policy.metric
&& payload.source === policy.source
&& payload.swapExcluded === true;
if (!metadataMatches || availableBytes === null) {
return unavailableGuard(spec, mode, metadataMatches ? "memavailable-unavailable" : "memory-snapshot-contract-invalid");
}
const comparator = mode === "manual-start" ? policy.manualComparator : policy.sentinelComparator;
const thresholdMatched = mode === "manual-start"
? availableBytes <= policy.thresholdBytes
: availableBytes < policy.thresholdBytes;
const status: WebProbeMemoryGuardStatus = thresholdMatched
? mode === "manual-start" ? "blocked" : "skipped"
: "allowed";
return {
ok: status === "allowed",
status,
eligible: status === "allowed",
startBlocked: status !== "allowed",
decision: status === "allowed" ? "start-observer" : mode === "manual-start" ? "gc-required" : "skip-observer",
reason: status === "allowed" ? null : "memavailable-threshold-matched",
mode,
policyConfigRef: webProbeMemoryStartGuardConfigRef,
memory: {
metric: policy.metric,
source: policy.source,
swapExcluded: policy.swapExcluded,
availableBytes,
availableHuman: humanBytes(availableBytes),
thresholdBytes: policy.thresholdBytes,
thresholdHuman: humanBytes(policy.thresholdBytes),
comparator,
thresholdMatched,
valuesRedacted: true,
},
gcNext: status === "blocked" && mode === "manual-start"
? { ...controlledGcNext(spec.nodeId), requiredBeforeStart: true }
: { ...controlledGcNext(spec.nodeId), requiredBeforeStart: false },
valuesRedacted: true,
};
}
export function runWebProbeMemoryGuard(
spec: HwlabRuntimeLaneSpec,
mode: WebProbeMemoryGuardMode,
execute: MemoryGuardExecutor = runCommand,
): Record<string, unknown> {
const policy = memoryPolicy(spec);
if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
const command = ["bun", "scripts/cli.ts", "gc", "remote", spec.nodeId, "memory"];
const result = execute(command, repoRoot, { timeoutMs: policy.snapshotTimeoutSeconds * 1000 });
const resolution = resolveCliChildJsonCommandResult({
result,
requestedStdoutType: `gc remote ${spec.nodeId} memory contract`,
acceptParsed: (value) => {
const payload = value.action === "gc remote memory" ? value : record(value.data);
return payload.action === "gc remote memory" && payload.providerId === spec.nodeId;
},
});
const resolvedPayload = resolution.parsed?.action === "gc remote memory"
? resolution.parsed
: record(resolution.parsed?.data);
const evaluated = evaluateWebProbeMemoryGuard(spec, mode, resolvedPayload);
return {
...evaluated,
snapshot: {
action: "gc remote memory",
command: command.join(" "),
exitCode: result.exitCode,
timedOut: result.timedOut,
parsed: resolution.parsed !== null,
diagnostics: resolution.diagnostics,
valuesRedacted: true,
},
};
}
@@ -20,6 +20,11 @@ const viewport = parseViewport(process.env.UNIDESK_WEB_PROBE_VIEWPORT || "1440x9
const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_PROBE_BROWSER_PROXY_MODE || "auto");
const playwrightProxy = proxyConfigFromEnv(baseUrl);
const chromiumLaunchOptions = chromiumLaunchOptionsForProxy(playwrightProxy);
const screenshotMode = requiredEnum(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_MODE, ["viewport"]);
const maxNetworkRequestEntries = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES, "UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_ENTRIES");
const maxNetworkRequestBytes = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES, "UNIDESK_WEB_PROBE_MAX_NETWORK_REQUEST_BYTES");
const maxNetworkFailureEntries = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES, "UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_ENTRIES");
const maxNetworkFailureBytes = requiredPositiveInteger(process.env.UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES, "UNIDESK_WEB_PROBE_MAX_NETWORK_FAILURE_BYTES");
const artifactRecords = [];
const readinessRecords = [];
const stepRecords = [];
@@ -106,6 +111,7 @@ try {
});
process.exitCode = 2;
} finally {
if (context) await context.close().catch(() => {});
if (browser) await browser.close().catch(() => {});
}
@@ -1314,36 +1320,54 @@ function createNetworkTracker(targetPage, apiPattern) {
const started = Date.now();
const requests = [];
const failures = [];
let requestBytes = 0;
let requestDropped = 0;
let failureBytes = 0;
let failureDropped = 0;
const matches = createApiMatcher(apiPattern);
const appendBounded = (items, item, kind) => {
const bytes = Buffer.byteLength(JSON.stringify(item), "utf8");
const entryLimit = kind === "request" ? maxNetworkRequestEntries : maxNetworkFailureEntries;
const byteLimit = kind === "request" ? maxNetworkRequestBytes : maxNetworkFailureBytes;
const usedBytes = kind === "request" ? requestBytes : failureBytes;
if (items.length >= entryLimit || usedBytes + bytes > byteLimit) {
if (kind === "request") requestDropped += 1;
else failureDropped += 1;
return;
}
items.push(item);
if (kind === "request") requestBytes += bytes;
else failureBytes += bytes;
};
const onRequest = (request) => {
if (!matches(request.url())) return;
requests.push({
appendBounded(requests, {
method: request.method(),
path: urlPath(request.url()),
atMs: Date.now() - started,
});
}, "request");
};
const onResponse = (response) => {
if (!matches(response.url())) return;
if (response.status() >= 400) {
failures.push({
appendBounded(failures, {
type: "response",
status: response.status(),
statusText: response.statusText(),
path: urlPath(response.url()),
atMs: Date.now() - started,
});
}, "failure");
}
};
const onRequestFailed = (request) => {
if (!matches(request.url())) return;
failures.push({
appendBounded(failures, {
type: "requestfailed",
method: request.method(),
path: urlPath(request.url()),
failureText: request.failure()?.errorText ?? null,
atMs: Date.now() - started,
});
}, "failure");
};
targetPage.on("request", onRequest);
targetPage.on("response", onResponse);
@@ -1356,6 +1380,10 @@ function createNetworkTracker(targetPage, apiPattern) {
apiFailureCount: failures.length,
apiRequests: requests.slice(0, 5),
apiFailures: failures.slice(0, 5),
apiRequestEvidenceBytes: requestBytes,
apiRequestEvidenceDropped: requestDropped,
apiFailureEvidenceBytes: failureBytes,
apiFailureEvidenceDropped: failureDropped,
}),
dispose: () => {
targetPage.off("request", onRequest);
@@ -1476,7 +1504,7 @@ async function captureFailureScreenshot(name = "failure.png") {
async function screenshotPage(targetPage, name = "screenshot.png", options = {}) {
const file = artifactPath(name);
await targetPage.screenshot({ path: file, fullPage: true, ...options });
await targetPage.screenshot({ path: file, fullPage: screenshotMode !== "viewport", ...options });
return recordArtifact(file, "screenshot");
}
@@ -1910,6 +1938,17 @@ function positiveInteger(raw, fallback) {
return value;
}
function requiredPositiveInteger(raw, name) {
const value = Number(raw);
if (!Number.isInteger(value) || value <= 0) throw new Error("missing or invalid " + name);
return value;
}
function requiredEnum(raw, allowed) {
if (!allowed.includes(raw)) throw new Error("missing or invalid screenshot mode");
return raw;
}
function artifactPath(name) {
const safe = String(name || "artifact")
.replace(/[^A-Za-z0-9._-]/gu, "_")
@@ -1920,7 +1959,7 @@ function artifactPath(name) {
async function screenshot(name = "screenshot.png", options = {}) {
const file = artifactPath(name);
await page.screenshot({ path: file, fullPage: true, ...options });
await page.screenshot({ path: file, fullPage: screenshotMode !== "viewport", ...options });
return recordArtifact(file, "screenshot");
}
@@ -439,6 +439,7 @@ function knownWebProbeFindingIds(): string[] {
"scripts/src/hwlab-node-web-observe-analyzer-window-page-source.ts",
"scripts/src/hwlab-node-web-observe-analyzer-workbench-triad-source.ts",
"scripts/src/hwlab-node-web-sentinel-p5-observe.ts",
"scripts/src/hwlab-node-web-sentinel-p5-observe-analysis.ts",
]) {
let source = "";
try {
File diff suppressed because it is too large Load Diff
@@ -6,7 +6,17 @@ import { join } from "node:path";
import { test } from "bun:test";
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
import { classifyQuickVerifyCleanupStep, quickVerifyCleanupFindings, readAnalysisSummaryFromWorkspace, sentinelServiceProxyInvocation } from "./hwlab-node-web-sentinel-p5-observe";
import { nodeWebObserveRunnerSource } from "./hwlab-node-web-observe-runner-source";
import {
classifyQuickVerifyCleanupStep,
createQuickVerifyObserverLifecycle,
quickVerifyCleanupFindings,
quickVerifyUnhandledExceptionSummary,
readAnalysisSummaryFromWorkspace,
sentinelChildStartMemorySkip,
sentinelServiceProxyInvocation,
stopQuickVerifyObserverAfterPartialStart,
} from "./hwlab-node-web-sentinel-p5-observe";
import { createWebProbeSentinelService } from "./hwlab-node-web-sentinel-service";
test("sentinel service proxy executes its script through trans standard input", () => {
@@ -60,6 +70,7 @@ test("recorded sentinel run is available through the latest report index", () =>
views: { summary: { renderedText: "indexed summary" } },
findingCount: 0,
artifactCount: 0,
updatedAt: "2026-07-13T00:00:00.000Z",
valuesRedacted: true,
});
const latest = service.report("summary", null);
@@ -91,6 +102,7 @@ test("latest report selects the newest stored run without a report SHA", () => {
views: { summary: { renderedText: "old indexed summary" } },
findingCount: 0,
artifactCount: 0,
updatedAt: "2026-07-13T00:00:00.000Z",
valuesRedacted: true,
});
service.recordRun({
@@ -102,6 +114,7 @@ test("latest report selects the newest stored run without a report SHA", () => {
findings: [{ id: "quick-verify-observer-start-failed", severity: "error", summary: "observer start failed" }],
findingCount: 1,
artifactCount: 0,
updatedAt: "2026-07-13T00:00:01.000Z",
valuesRedacted: true,
});
@@ -116,7 +129,6 @@ test("latest report selects the newest stored run without a report SHA", () => {
rmSync(stateRoot, { recursive: true, force: true });
}
});
function forceStopStep(observer: Record<string, unknown>): Record<string, unknown> {
return {
phase: "observe-stop-after-terminal",
@@ -224,3 +236,146 @@ test("quick verify cleanup emits evidence-gap code when stopped state cannot be
assert.equal(findings[0]?.id, "observer-cleanup-evidence-missing");
assert.match(String(findings[0]?.evidenceSummary), /missing=aliveAfter/u);
});
test("quick verify observer lifecycle stops once on success, failure, and exception fallback", () => {
const cases = [
{ name: "success", initialPhase: "observe-stop-after-terminal", expectedPhase: "observe-stop-after-terminal" },
{ name: "failure", initialPhase: "observe-stop-after-failure", expectedPhase: "observe-stop-after-failure" },
{ name: "exception-fallback", initialPhase: null, expectedPhase: "observe-stop-after-finally" },
] as const;
for (const scenario of cases) {
const calls: { observerId: string; phase: string }[] = [];
const lifecycle = createQuickVerifyObserverLifecycle("webobs-fixture", (observerId, phase) => {
calls.push({ observerId, phase });
return { phase, ok: true, observerId };
});
if (scenario.initialPhase !== null) lifecycle.stop(scenario.initialPhase);
const cleanupStep = lifecycle.stop("observe-stop-after-finally");
assert.deepEqual(calls, [{ observerId: "webobs-fixture", phase: scenario.expectedPhase }], scenario.name);
assert.equal(cleanupStep.phase, scenario.expectedPhase, scenario.name);
assert.strictEqual(cleanupStep, lifecycle.cleanupStep, scenario.name);
}
});
test("quick verify observer lifecycle retries a failed stop again from the bounded finally fallback", () => {
let calls = 0;
const lifecycle = createQuickVerifyObserverLifecycle("webobs-stop-throws", () => {
calls += 1;
throw new Error("sensitive-stop-error");
});
const first = lifecycle.stop("observe-stop-after-failure");
const fallback = lifecycle.stop("observe-stop-after-finally");
assert.equal(calls, 4);
assert.notStrictEqual(first, fallback);
assert.equal(first.failure, "observer-stop-unhandled-exception");
assert.equal(Array.isArray(fallback.stopAttempts), true);
assert.equal((fallback.stopAttempts as unknown[]).length, 4);
assert.equal(JSON.stringify(first).includes("sensitive-stop-error"), false);
});
test("quick verify observer lifecycle retries a failed force-stop and caches the successful fallback", () => {
let calls = 0;
const lifecycle = createQuickVerifyObserverLifecycle("webobs-stop-retry", (_observerId, phase) => {
calls += 1;
return { phase, ok: calls === 2, observerId: "webobs-stop-retry" };
});
const first = lifecycle.stop("observe-stop-after-failure");
const cached = lifecycle.stop("observe-stop-after-finally");
assert.equal(calls, 2);
assert.equal(first.ok, true);
assert.strictEqual(first, cached);
assert.equal(Array.isArray(first.stopAttempts), true);
});
test("quick verify cleans a partial start only when start output contains an observer id", () => {
const calls: { observerId: string; phase: string }[] = [];
const stopObserver = (observerId: string, phase: string) => {
calls.push({ observerId, phase });
return { phase, ok: true, observerId };
};
assert.equal(stopQuickVerifyObserverAfterPartialStart(null, stopObserver), null);
const cleanupStep = stopQuickVerifyObserverAfterPartialStart("webobs-partial", stopObserver);
assert.deepEqual(calls, [{ observerId: "webobs-partial", phase: "observe-stop-after-start-failure" }]);
assert.equal(cleanupStep?.ok, true);
});
test("quick verify unexpected exception evidence is bounded and redacted", () => {
const secret = "sensitive-upstream-message";
const summary = quickVerifyUnhandledExceptionSummary(new Error(secret));
const serialized = JSON.stringify(summary);
assert.equal(summary.errorName, "Error");
assert.match(String(summary.errorMessageSha256), /^[a-f0-9]{64}$/u);
assert.equal(summary.errorMessageTruncated, false);
assert.equal(serialized.includes(secret), false);
});
test("sentinel preserves a child launch-time memory race as skipped instead of observe-start-failed", () => {
const skip = sentinelChildStartMemorySkip({
ok: true,
command: "web-probe observe start",
data: {
ok: true,
status: "skipped-wait-next-round",
observerCreated: false,
memoryGuard: {
mode: "sentinel-cadence",
status: "skipped",
memory: {
metric: "MemAvailable",
source: "/proc/meminfo",
swapExcluded: true,
availableBytes: 4 * 1024 ** 3 - 1,
thresholdBytes: 4 * 1024 ** 3,
comparator: "less-than",
},
},
},
});
assert.equal(skip?.status, "skipped-wait-next-round");
assert.equal(skip?.decision, "skip-observer");
assert.equal(skip?.observerCreated, false);
assert.equal((skip?.memoryGuard as Record<string, unknown>).mode, "sentinel-cadence");
assert.equal(sentinelChildStartMemorySkip({ data: { status: "blocked", observerCreated: false } }), null);
});
test("web observe runner records finite lifecycle policy and closes the browser", () => {
const source = nodeWebObserveRunnerSource();
assert.match(source, /sampling:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
assert.match(source, /observerLifecycle:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
assert.match(source, /if \(maxRunMs > 0 && Date\.now\(\) - startedAtMs >= maxRunMs\)/u);
assert.match(source, /const maxScreenshots = requiredBoundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS/u);
assert.match(source, /const maxPerformanceEntries = requiredBoundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES/u);
assert.match(source, /const maxRealtimeEventBodyBytes = requiredBoundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES/u);
assert.match(source, /while \(browserProcessHistory\.length > maxBrowserProcessHistoryEntries\)/u);
assert.match(source, /while \(browserFreezeSignalHistory\.length > maxBrowserFreezeSignalHistoryEntries\)/u);
assert.match(source, /if \(screenshotCount >= maxScreenshots\)/u);
assert.match(source, /const maxBytes = responseBodyMaxBytes/u);
assert.match(source, /function responseBodyReadContract\(response, maxBytes = responseBodyMaxBytes\)/u);
assert.match(source, /contentEncoding[\s\S]*?reason: "encoded-body-size-unbounded"/u);
assert.match(source, /!Number\.isFinite\(contentLength\)[\s\S]*?reason: "content-length-unavailable"/u);
assert.match(source, /bodyReadSkipped: body\.error\?\.reason \|\| "bounded-response-read-failed"/u);
assert.match(source, /\.slice\(0, maxProcessScanEntries\)/u);
assert.match(source, /if \(items\.length > maxEntries\)[\s\S]*?items\.splice/u);
assert.match(source, /process\.once\(signal,[\s\S]*?closeBrowserResources\("signal:" \+ signal\)/u);
assert.match(source, /finally \{[\s\S]*?await closeBrowserResources/u);
assert.match(source, /closingObserverPage\.close\(\)[\s\S]*?closingPage\.close\(\)[\s\S]*?closingContext\.close\(\)[\s\S]*?closingBrowser\.close\(\)/u);
const directory = mkdtempSync(join(tmpdir(), "web-observe-runner-check-"));
try {
const runner = join(directory, "observer-runner.mjs");
writeFileSync(runner, source);
const syntax = spawnSync("node", ["--check", runner], { encoding: "utf8" });
assert.equal(syntax.status, 0, syntax.stderr);
} finally {
rmSync(directory, { recursive: true, force: true });
}
});
File diff suppressed because it is too large Load Diff
+16 -7
View File
@@ -9,6 +9,7 @@ import { startJob } from "./jobs";
import type { RenderedCliResult } from "./output";
import { runWebProbeRemoteArtifactJob } from "./web-probe-remote-artifact";
import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-config-ref";
import { runWebProbeMemoryGuard, webProbeGuardedLaunchShell, webProbeScreenshotFullPage } from "./hwlab-node-web-probe-memory-guard";
import type { SentinelCicdState, WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
import {
clipTail,
@@ -1144,11 +1145,13 @@ export function runSentinelDashboard(state: SentinelCicdState, options: Extract<
}
export function probeSentinelDashboardBrowser(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "dashboard" }>): Record<string, unknown> {
const memoryGuard = runWebProbeMemoryGuard(state.spec, "sentinel-cadence");
if (memoryGuard.status !== "allowed") return memoryGuard;
const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, "");
const dashboardUrl = sentinelDashboardUrl(publicBaseUrl, options.runId, options.errorId);
const [widthRaw, heightRaw] = options.viewport.split("x");
const screenshotName = options.action === "screenshot" ? dashboardScreenshotName(options, state) : "";
const script = [
const browserScript = [
"set -eu",
`export UNIDESK_SENTINEL_DASHBOARD_URL=${shellQuote(dashboardUrl)}`,
`export UNIDESK_SENTINEL_DASHBOARD_SCREENSHOT="$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR"/${shellQuote(screenshotName)}`,
@@ -1157,7 +1160,7 @@ export function probeSentinelDashboardBrowser(state: SentinelCicdState, options:
`export UNIDESK_SENTINEL_DASHBOARD_WIDTH=${shellQuote(widthRaw ?? "1440")}`,
`export UNIDESK_SENTINEL_DASHBOARD_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
`export UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
`export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
`export UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE=${shellQuote(webProbeScreenshotFullPage(state.spec, options.fullPage) ? "1" : "0")}`,
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID=${shellQuote(state.sentinelId)}`,
`export UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX=${shellQuote(stringAtNullable(state.publicExposure, "routePrefix") ?? "/")}`,
`export UNIDESK_SENTINEL_DASHBOARD_RUN_ID=${shellQuote(options.runId ?? "")}`,
@@ -1178,6 +1181,7 @@ export function probeSentinelDashboardBrowser(state: SentinelCicdState, options:
"WEB_PROBE_SENTINEL_DASHBOARD_JS",
"node \"$UNIDESK_WEB_PROBE_ARTIFACT_REMOTE_DIR/web-probe-sentinel-dashboard.mjs\"",
].join("\n");
const script = webProbeGuardedLaunchShell(state.spec, "sentinel-cadence", browserScript);
const route = `${state.spec.nodeId}:${state.spec.workspace}`;
const job = runWebProbeRemoteArtifactJob({
route,
@@ -1263,7 +1267,7 @@ const triggerManual = process.env.UNIDESK_SENTINEL_DASHBOARD_TRIGGER === "1";
const width = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_WIDTH || 1440);
const height = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_HEIGHT || 900);
const timeout = Number(process.env.UNIDESK_SENTINEL_DASHBOARD_TIMEOUT_MS || 30000);
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE !== "0";
const fullPage = process.env.UNIDESK_SENTINEL_DASHBOARD_FULL_PAGE === "1";
const executablePath = process.env.UNIDESK_SENTINEL_DASHBOARD_EXECUTABLE_PATH || "";
const expectedSentinelId = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_SENTINEL_ID || "";
const expectedRoutePrefix = process.env.UNIDESK_SENTINEL_DASHBOARD_EXPECTED_ROUTE_PREFIX || "/";
@@ -1275,12 +1279,15 @@ if (!url) throw new Error("missing dashboard URL");
const consoleMessages = [];
const pageErrors = [];
const requestFailures = [];
const browser = await chromium.launch({
let browser = null;
let context = null;
try {
browser = await chromium.launch({
headless: true,
args: ["--disable-gpu", "--no-sandbox"],
...(executablePath ? { executablePath } : {}),
});
const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
const page = await context.newPage();
page.on("console", (message) => {
if (consoleMessages.length < 30) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 300) });
@@ -1677,8 +1684,10 @@ console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({
valuesRedacted: true,
}));
await context.close().catch(() => {});
await browser.close().catch(() => {});
} finally {
if (context) await context.close().catch(() => {});
if (browser) await browser.close().catch(() => {});
}
`;
}
@@ -562,6 +562,7 @@ function buildObserveCommandPlan(config: WebProbeSentinelServiceConfig, scenario
"--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")),
"--max-run-seconds", String(numberAt(scenario, "maxRunSeconds")),
"--command-timeout-seconds", "55",
"--sentinel-cadence",
];
const viewport = stringOrNull(scenario.viewport);
if (viewport !== null) startArgv.push("--viewport", viewport);
@@ -2727,7 +2728,7 @@ function reportRunView(config: WebProbeSentinelServiceConfig, db: Database, view
return { ok: false, error: "unsupported-report-view", view, valuesRedacted: true };
}
const row = runId === null
? db.query("SELECT * FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY updated_at DESC LIMIT 1").get(config.sentinelId, config.node, config.lane) as Record<string, unknown> | null
? db.query("SELECT * FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY updated_at DESC, rowid DESC LIMIT 1").get(config.sentinelId, config.node, config.lane) as Record<string, unknown> | null
: readRunRow(config, db, runId);
if (row === null) return { ok: false, error: "report-run-missing", runId, view, valuesRedacted: true };
const selectedRunId = stringOrNull(row.id);
+1
View File
@@ -188,6 +188,7 @@ export interface NodeWebProbeObserveOptions extends WebProbeOriginSelection {
targetPath: string;
viewport: string;
browserProxyMode: WebProbeBrowserProxyMode;
memoryGuardMode: "manual-start" | "sentinel-cadence";
sampleIntervalMs: number;
screenshotIntervalMs: number;
observerRefreshIntervalMs: number;
@@ -25,6 +25,7 @@ import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapp
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell } from "../hwlab-node-web-probe-memory-guard";
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
import type { RenderedCliResult } from "../output";
@@ -537,7 +538,9 @@ export function runNodeWebProbeScript(
const commandLabel = options.commandLabel ?? `web-probe script --node ${options.node} --lane ${options.lane}${webProbeSemanticOriginArgs(options)}`;
const commandGovernance = webProbeScriptCommandGovernance(options);
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.username ?? secretSpec.bootstrapAdminUsername, material.password ?? "", webProbeProxy, spec.webProbe?.playwrightBrowsersPath);
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
if (memoryGuard.status !== "allowed") return memoryGuard;
const script = nodeWebProbeScriptRemoteShell(options, spec, secretSpec, material.username ?? secretSpec.bootstrapAdminUsername, material.password ?? "", webProbeProxy, spec.webProbe?.playwrightBrowsersPath);
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const commandTimedOut = result.timedOut || result.exitCode === 124;
const runPaths = webProbeScriptRunPathsFromStderr(result.stderr);
@@ -740,7 +743,7 @@ function webProbeSemanticOriginArgs(options: Pick<NodeWebProbeScriptOptions, "or
return options.originName === "internal" || options.originName === "public" ? ` --origin ${options.originName}` : "";
}
export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, username: string, password: string, webProbeProxy: NodeWebProbeHostProxyEnv, playwrightBrowsersPath: string | undefined): string {
export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, spec: HwlabRuntimeLaneSpec, secretSpec: RuntimeSecretSpec, username: string, password: string, webProbeProxy: NodeWebProbeHostProxyEnv, playwrightBrowsersPath: string | undefined): string {
const userScriptB64 = Buffer.from(options.scriptText, "utf8").toString("base64");
const runnerB64 = Buffer.from(nodeWebProbeScriptRunnerSource(), "utf8").toString("base64");
return [
@@ -756,8 +759,9 @@ export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions
"runner=\"$run_dir/runner.mjs\"",
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$user_script" ${shellQuote(userScriptB64)}`,
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
[
webProbeGuardedLaunchShell(spec, "manual-start", [
...webProbeProxy.envAssignments,
...webProbeBrowserEvidenceEnvAssignments(spec),
...(playwrightBrowsersPath === undefined ? [] : [
`PLAYWRIGHT_BROWSERS_PATH=${shellQuote(playwrightBrowsersPath)}`,
]),
@@ -770,7 +774,7 @@ export function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions
`UNIDESK_WEB_PROBE_VIEWPORT=${shellQuote(options.viewport)}`,
`UNIDESK_WEB_PROBE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
"node \"$runner\"",
].join(" "),
].join(" ")),
].join("\n");
}
@@ -3,6 +3,7 @@
import { randomBytes } from "node:crypto";
import { resolveCliChildJsonCommandResult, type CliChildJsonResolution } from "../cli-child-json-recovery";
import type { HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard } from "../hwlab-node-web-probe-memory-guard";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
import { withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
@@ -454,6 +455,134 @@ console.log(JSON.stringify({
`;
}
export function nodeWebObserveFinalLaunchGuardSource(): string {
return String.raw`
const fs = require("fs");
const thresholdBytes = Number(process.argv[2]);
const comparator = process.argv[3];
const mode = process.argv[4];
const jobId = process.argv[5];
const stateDir = process.argv[6];
const source = "/proc/meminfo";
function finish(reason, availableBytes) {
const valid = Number.isSafeInteger(availableBytes) && availableBytes >= 0;
const thresholdMatched = valid
? comparator === "less-than" ? availableBytes < thresholdBytes : availableBytes <= thresholdBytes
: null;
const blocked = reason !== null || thresholdMatched === true;
const cadenceSkip = blocked && mode === "sentinel-cadence";
const payload = {
ok: !blocked || cadenceSkip,
command: "web-probe-observe start",
status: blocked ? cadenceSkip ? "skipped-wait-next-round" : "blocked" : "allowed",
reason: reason || (thresholdMatched ? "memavailable-threshold-matched" : null),
failureKind: blocked ? "web-probe-final-memory-start-guard" : null,
jobId,
stateDir,
observerCreated: false,
browserCreated: false,
startBlocked: blocked,
decision: blocked ? cadenceSkip ? "skip-observer" : "gc-required" : "start-observer",
mutation: blocked,
stateArtifactCreated: blocked,
memoryGuard: {
phase: "remote-launch-critical-section",
metric: "MemAvailable",
source,
swapExcluded: true,
availableBytes: valid ? availableBytes : null,
thresholdBytes,
comparator,
thresholdMatched,
},
observedAt: new Date().toISOString(),
valuesRedacted: true,
};
console.log(JSON.stringify(payload));
process.exit(blocked ? 42 : 0);
}
if (!Number.isSafeInteger(thresholdBytes) || thresholdBytes <= 0) finish("memory-threshold-invalid", null);
if (comparator !== "less-than" && comparator !== "less-than-or-equal") finish("memory-comparator-invalid", null);
let text;
try { text = fs.readFileSync(source, "utf8"); } catch { finish("proc-meminfo-read-failed", null); }
const match = text.match(/^MemAvailable:\s+(\d+)\s+kB$/m);
if (!match) finish("memavailable-missing", null);
const availableBytes = Number(match[1]) * 1024;
finish(null, availableBytes);
`;
}
export function nodeWebObserveLaunchReadinessSource(): string {
return String.raw`
const fs = require("fs");
const runnerPid = Number(process.argv[2]);
const stateDir = process.argv[3];
const expectedJobId = process.argv[4];
const timeoutMs = Number(process.argv[5]);
const pollMs = Number(process.argv[6]);
const deadline = Date.now() + timeoutMs;
const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
const readJson = (name) => { try { return JSON.parse(fs.readFileSync(stateDir + "/" + name, "utf8")); } catch { return null; } };
function procRows() {
const rows = new Map();
let names = [];
try { names = fs.readdirSync("/proc").filter((name) => /^\d+$/.test(name)); } catch { return rows; }
for (const name of names) {
try {
const stat = fs.readFileSync("/proc/" + name + "/stat", "utf8");
const close = stat.lastIndexOf(")");
const fields = stat.slice(close + 2).trim().split(/\s+/);
const command = fs.readFileSync("/proc/" + name + "/cmdline").toString("utf8").replace(/\0/g, " ");
rows.set(Number(name), { pid: Number(name), ppid: Number(fields[1]), command });
} catch {}
}
return rows;
}
function descendants(rows) {
const selected = new Set([runnerPid]);
let changed = true;
while (changed) {
changed = false;
for (const row of rows.values()) if (!selected.has(row.pid) && selected.has(row.ppid)) { selected.add(row.pid); changed = true; }
}
return [...selected].map((pid) => rows.get(pid)).filter(Boolean);
}
function matchingStateRecord(name, value) {
if (!value || typeof value !== "object") return null;
const status = String(value.status || "");
if (!/^(starting|running|ready)$/i.test(status)) return null;
if (String(value.jobId || "") !== expectedJobId || Number(value.pid) !== runnerPid) return null;
return { name, status, jobId: String(value.jobId), pid: Number(value.pid) };
}
while (Date.now() <= deadline) {
const rows = procRows();
const runnerAlive = rows.has(runnerPid);
const children = runnerAlive ? descendants(rows) : [];
const browserChild = children.find((row) => row.pid !== runnerPid && /(?:^|[ /])(chrome|chromium)(?:[ /]|$)/i.test(row.command));
const heartbeat = readJson("heartbeat.json");
const manifest = readJson("manifest.json");
const stateRecord = matchingStateRecord("heartbeat.json", heartbeat) || matchingStateRecord("manifest.json", manifest);
if (runnerAlive && browserChild && stateRecord) {
console.log(JSON.stringify({ ok: true, expectedJobId, runnerPid, runnerAlive: true, browserPid: browserChild.pid, stateRecord, readiness: "current-state-and-browser-descendant", observedAt: new Date().toISOString(), valuesRedacted: true }));
process.exit(0);
}
if (!runnerAlive) {
console.log(JSON.stringify({ ok: false, reason: "runner-exited-before-ready", expectedJobId, runnerPid, runnerAlive: false, stateRecord, observedAt: new Date().toISOString(), valuesRedacted: true }));
process.exit(43);
}
pause(pollMs);
}
console.log(JSON.stringify({ ok: false, reason: "launch-readiness-timeout", expectedJobId, runnerPid, runnerAlive: fs.existsSync("/proc/" + runnerPid), observedAt: new Date().toISOString(), valuesRedacted: true }));
process.exit(44);
`;
}
export function runNodeWebProbeObserveStart(
options: NodeWebProbeObserveOptions,
spec: HwlabRuntimeLaneSpec,
@@ -461,6 +590,29 @@ export function runNodeWebProbeObserveStart(
material: BootstrapAdminPasswordMaterial,
credential: Record<string, unknown>,
): Record<string, unknown> | RenderedCliResult {
const memoryGuard = runWebProbeMemoryGuard(spec, options.memoryGuardMode);
if (memoryGuard.status !== "allowed") {
const cadenceSkip = options.memoryGuardMode === "sentinel-cadence" && memoryGuard.status === "skipped";
return {
ok: cadenceSkip,
status: cadenceSkip ? "skipped-wait-next-round" : "blocked",
failureKind: "web-probe-memory-start-guard",
command: `web-probe observe start --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
workspace: spec.workspace,
observerCreated: false,
startBlocked: true,
decision: cadenceSkip ? "skip-observer" : "gc-required",
mutation: false,
memoryGuard,
valuesRedacted: true,
};
}
const observerLifecycle = spec.webProbe?.resourcePolicy?.observerLifecycle;
const finalMemoryPolicy = spec.webProbe?.resourcePolicy?.memoryStartGuard;
if (observerLifecycle === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML");
if (finalMemoryPolicy === undefined) throw new Error("web-probe memory start policy is missing from the owning YAML");
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
const day = timestamp.slice(0, 8);
@@ -494,6 +646,28 @@ export function runNodeWebProbeObserveStart(
`UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`,
`UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`,
`UNIDESK_WEB_OBSERVE_MAX_RUN_MS=${shellQuote(String(options.maxRunSeconds > 0 ? options.maxRunSeconds * 1000 : 0))}`,
`UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS=${shellQuote(String(observerLifecycle.maxScreenshots))}`,
`UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES=${shellQuote(String(observerLifecycle.maxPerformanceEntries))}`,
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserProcessHistoryEntries))}`,
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserFreezeSignalHistoryEntries))}`,
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES=${shellQuote(String(observerLifecycle.maxRealtimeEventEntries))}`,
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES=${shellQuote(String(observerLifecycle.maxRealtimeEventBodyBytes))}`,
`UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS=${shellQuote(String(observerLifecycle.maxDomEvidenceRows))}`,
`UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES=${shellQuote(String(observerLifecycle.maxProcessScanEntries))}`,
`UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS=${shellQuote(String(observerLifecycle.maxPerformanceCaptureSeconds))}`,
`UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxResponseBodyBytes))}`,
`UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxWebPerformanceBodyBytes))}`,
`UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES=${shellQuote(String(observerLifecycle.maxJsonlQueueEntries))}`,
`UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES=${shellQuote(String(observerLifecycle.maxJsonlQueueBytes))}`,
`UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES=${shellQuote(String(observerLifecycle.maxJsonlLineBytes))}`,
`UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS=${shellQuote(String(observerLifecycle.jsonlFlushTimeoutMs))}`,
`UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT=${shellQuote(String(observerLifecycle.maxResponseBodyInFlight))}`,
`UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS=${shellQuote(String(observerLifecycle.maxPassiveListenerTasks))}`,
`UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS=${shellQuote(String(observerLifecycle.passiveTaskFlushTimeoutMs))}`,
`UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS=${shellQuote(String(observerLifecycle.closeStepTimeoutMs))}`,
`UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY=${shellQuote(String(observerLifecycle.maxProcessReadConcurrency))}`,
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES=${shellQuote(String(observerLifecycle.maxRealtimeTotalEntries))}`,
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES=${shellQuote(String(observerLifecycle.maxRealtimeTotalBodyBytes))}`,
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
@@ -511,6 +685,26 @@ export function runNodeWebProbeObserveStart(
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`,
]),
].join(" ");
const finalComparator = options.memoryGuardMode === "sentinel-cadence"
? finalMemoryPolicy.sentinelComparator
: finalMemoryPolicy.manualComparator;
const finalGuardFailure = (reason: string) => JSON.stringify({
ok: false,
command: "web-probe-observe start",
status: "blocked",
reason,
failureKind: "web-probe-final-memory-start-guard",
jobId,
stateDir,
observerCreated: false,
startBlocked: true,
mutation: true,
stateArtifactCreated: true,
browserCreated: false,
valuesRedacted: true,
});
const launchFailureOutputSource = "const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const guard=read('launch-guard.json'); const readiness=read('launch-ready.json'); const pid=Number(process.argv[3]); const bool=(v)=>v==='true'; console.log(JSON.stringify({ok:false,command:'web-probe-observe start',status:'blocked',reason:'launch-readiness-unconfirmed',failureKind:'web-probe-launch-readiness',jobId:process.argv[2],stateDir:dir,observerCreated:false,browserCreated:false,browserCreationAttempted:true,startBlocked:true,mutation:true,stateArtifactCreated:true,memoryGuard:guard&&guard.memoryGuard||null,finalLaunchGuard:guard,launchReadiness:readiness,cleanup:{attempted:true,processGroupId:pid,termSent:bool(process.argv[4]),killSent:bool(process.argv[5]),aliveAfter:bool(process.argv[6]),completed:!bool(process.argv[6]),observedAt:new Date().toISOString(),valuesRedacted:true},valuesRedacted:true},null,2))";
const cleanupPollCount = Math.ceil(finalMemoryPolicy.launchReadyTimeoutSeconds * 1000 / finalMemoryPolicy.launchReadyPollMilliseconds);
const script = [
"set -eu",
`state_dir=${shellQuote(stateDir)}`,
@@ -524,16 +718,71 @@ export function runNodeWebProbeObserveStart(
"node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"",
"rm -f \"$runner_b64\"",
"chmod 700 \"$runner\"",
`if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & else nohup env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & fi`,
`launch_lock=${shellQuote(finalMemoryPolicy.launchLockPath)}`,
`launch_guard_result="$state_dir/launch-guard.json"`,
`launch_ready_result="$state_dir/launch-ready.json"`,
`if ! command -v flock >/dev/null 2>&1; then printf '%s\\n' ${shellQuote(finalGuardFailure("host-flock-unavailable"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
`if ! command -v setsid >/dev/null 2>&1; then printf '%s\\n' ${shellQuote(finalGuardFailure("host-setsid-unavailable"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
`exec 9>"$launch_lock"`,
`if ! flock -w ${shellQuote(String(finalMemoryPolicy.launchLockTimeoutSeconds))} 9; then printf '%s\\n' ${shellQuote(finalGuardFailure("launch-lock-timeout"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
"trap 'flock -u 9 2>/dev/null || true' EXIT",
"set +e",
`node - ${shellQuote(String(finalMemoryPolicy.thresholdBytes))} ${shellQuote(finalComparator)} ${shellQuote(options.memoryGuardMode)} ${shellQuote(jobId)} ${shellQuote(stateDir)} >"$launch_guard_result" <<'UNIDESK_WEB_OBSERVE_FINAL_GUARD'`,
nodeWebObserveFinalLaunchGuardSource(),
"UNIDESK_WEB_OBSERVE_FINAL_GUARD",
"launch_guard_rc=$?",
"set -e",
"if [ \"$launch_guard_rc\" -ne 0 ]; then cat \"$launch_guard_result\"; flock -u 9; exit 0; fi",
`setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null 9>&- &`,
"pid=$!",
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
"sleep 1",
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
"set +e",
`node - "$pid" "$state_dir" ${shellQuote(jobId)} ${shellQuote(String(finalMemoryPolicy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds))} >"$launch_ready_result" <<'UNIDESK_WEB_OBSERVE_LAUNCH_READY'`,
nodeWebObserveLaunchReadinessSource(),
"UNIDESK_WEB_OBSERVE_LAUNCH_READY",
"launch_ready_rc=$?",
"set -e",
`if [ "$launch_ready_rc" -ne 0 ]; then term_sent=false; if kill -TERM -- -"$pid" 2>/dev/null; then term_sent=true; fi; i=0; while [ "$i" -lt ${shellQuote(String(cleanupPollCount))} ] && kill -0 -- -"$pid" 2>/dev/null; do sleep ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds / 1000))}; i=$((i+1)); done; kill_sent=false; if kill -0 -- -"$pid" 2>/dev/null; then if kill -KILL -- -"$pid" 2>/dev/null; then kill_sent=true; fi; fi; i=0; while [ "$i" -lt ${shellQuote(String(cleanupPollCount))} ] && kill -0 -- -"$pid" 2>/dev/null; do sleep ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds / 1000))}; i=$((i+1)); done; alive_after=false; if kill -0 -- -"$pid" 2>/dev/null; then alive_after=true; fi; node -e ${shellQuote(launchFailureOutputSource)} "$state_dir" ${shellQuote(jobId)} "$pid" "$term_sent" "$kill_sent" "$alive_after"; flock -u 9; trap - EXIT; exit 0; fi`,
"flock -u 9",
"trap - EXIT",
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),finalLaunchGuard:read('launch-guard.json'),launchReadiness:read('launch-ready.json'),observerCreated:true,browserCreated:true,observerMutation:true,stateArtifactMutation:true,statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const startResolution = resolveWebObserveActionJson(result, "start");
const started = startResolution.parsed;
if (started?.observerCreated === false && started.startBlocked === true) {
const finalSnapshot = record(started.memoryGuard);
const finalMemoryGuard = evaluateWebProbeMemoryGuard(spec, options.memoryGuardMode, {
ok: finalSnapshot.metric === "MemAvailable" && finalSnapshot.source === "/proc/meminfo" && finalSnapshot.swapExcluded === true,
action: "gc remote memory",
metric: finalSnapshot.metric,
source: finalSnapshot.source,
swapExcluded: finalSnapshot.swapExcluded,
memory: { availableBytes: finalSnapshot.availableBytes },
});
return {
...started,
node: options.node,
lane: options.lane,
workspace: spec.workspace,
initialMemoryGuard: memoryGuard,
memoryGuard: finalMemoryGuard,
finalLaunchGuard: started.finalLaunchGuard ?? started,
valuesRedacted: true,
};
}
const startOk = result.exitCode === 0 && started?.ok === true;
const allowedFinalSnapshot = record(record(started?.finalLaunchGuard).memoryGuard);
const effectiveMemoryGuard = allowedFinalSnapshot.metric === "MemAvailable"
? evaluateWebProbeMemoryGuard(spec, options.memoryGuardMode, {
ok: true,
action: "gc remote memory",
metric: allowedFinalSnapshot.metric,
source: allowedFinalSnapshot.source,
swapExcluded: allowedFinalSnapshot.swapExcluded,
memory: { availableBytes: allowedFinalSnapshot.availableBytes },
})
: memoryGuard;
const recovery = startOk
? null
: readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30));
@@ -592,6 +841,10 @@ export function runNodeWebProbeObserveStart(
alertThresholds,
browserFreezePolicy,
projectManagement,
initialMemoryGuard: memoryGuard,
memoryGuard: effectiveMemoryGuard,
finalLaunchGuard: started?.finalLaunchGuard ?? null,
launchReadiness: started?.launchReadiness ?? null,
targetPath: options.targetPath,
id: observerId,
degradedReason,
@@ -171,6 +171,72 @@ test("observe start selects the YAML public origin semantically", () => {
assert.equal(options.browserProxyModeSource, "yaml-origin");
});
test("observe start inherits finite observer lifecycle limits from the owning YAML", () => {
const options = parseNodeWebProbeObserveOptions(
["start"],
"NC01",
"v03",
spec,
null,
null,
);
assert.equal(options.maxRunSeconds, 3600);
assert.equal(options.maxSamples, 720);
assert.equal(options.memoryGuardMode, "manual-start");
assert.ok(options.maxRunSeconds > 0);
assert.ok(options.maxSamples > 0);
assert.deepEqual(spec.webProbe?.resourcePolicy?.observerLifecycle, {
maxRunSeconds: 3600,
maxSamples: 720,
maxScreenshots: 24,
maxPerformanceEntries: 1000,
maxBrowserProcessHistoryEntries: 360,
maxBrowserFreezeSignalHistoryEntries: 120,
maxRealtimeEventEntries: 256,
maxRealtimeEventBodyBytes: 65536,
maxDomEvidenceRows: 1000,
maxProcessScanEntries: 8192,
maxPerformanceCaptureSeconds: 30,
maxResponseBodyBytes: 524288,
maxWebPerformanceBodyBytes: 65536,
maxJsonlQueueEntries: 512,
maxJsonlQueueBytes: 4194304,
maxJsonlLineBytes: 262144,
jsonlFlushTimeoutMs: 5000,
maxResponseBodyInFlight: 2,
maxPassiveListenerTasks: 64,
passiveTaskFlushTimeoutMs: 5000,
closeStepTimeoutMs: 5000,
maxProcessReadConcurrency: 32,
maxRealtimeTotalEntries: 512,
maxRealtimeTotalBodyBytes: 4194304,
});
});
test("sentinel observe start preserves the cadence comparator through the child CLI", () => {
const options = parseNodeWebProbeObserveOptions(
["start", "--sentinel-cadence"],
"NC01",
"v03",
spec,
null,
null,
);
assert.equal(options.memoryGuardMode, "sentinel-cadence");
});
test("observe start cannot disable or exceed the YAML lifecycle caps", () => {
assert.throws(
() => parseNodeWebProbeObserveOptions(["start", "--max-run-seconds", "0"], "NC01", "v03", spec, null, null),
/must be a positive integer/u,
);
assert.throws(
() => parseNodeWebProbeObserveOptions(["start", "--max-samples", "721"], "NC01", "v03", spec, null, null),
/exceeds the owning YAML observer lifecycle cap 720/u,
);
});
test("observe start refuses URL-based internal/public selection ambiguity", () => {
assert.throws(
() => parseNodeWebProbeObserveOptions(
+43 -14
View File
@@ -26,6 +26,7 @@ import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapp
import { runWebProbeSentinelCommand, type WebProbeSentinelDashboardAction, type WebProbeSentinelOptions, type WebProbeSentinelReportView, type WebProbeSentinelSourceAuthority, type WebProbeSentinelSourceOverrideOptions } from "../hwlab-node-web-sentinel-cicd";
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
import { runWebProbeMemoryGuard, webProbeBrowserEvidenceEnvAssignments, webProbeGuardedLaunchShell, webProbeScreenshotFullPage } from "../hwlab-node-web-probe-memory-guard";
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
import { runWebProbeRemoteArtifactJob } from "../web-probe-remote-artifact";
@@ -483,7 +484,7 @@ export function parseNodeWebProbeObserveOptions(
"--workspace-root",
"--workspace-root-ref",
"--root",
]), new Set(["--force", "--full", "--raw", "--text-stdin", "--require-composer-ready", "--wait-project-management-ready", "--blocking", "--non-blocking", "--dry-run", "--confirm"]));
]), new Set(["--force", "--full", "--raw", "--text-stdin", "--require-composer-ready", "--wait-project-management-ready", "--blocking", "--non-blocking", "--dry-run", "--confirm", "--sentinel-cadence"]));
const commandTypeRaw = optionValue(args, "--type") ?? null;
const commandType = commandTypeRaw === null ? null : parseNodeWebProbeObserveCommandType(commandTypeRaw);
const stateDir = optionValue(args, "--state-dir") ?? indexed?.stateDir ?? null;
@@ -528,6 +529,9 @@ export function parseNodeWebProbeObserveOptions(
if (observeActionRaw !== "start" && (requestedOrigin !== undefined || customUrl !== undefined || browserProxyModeRaw !== undefined)) {
throw new Error("web-probe observe --origin/--url/--browser-proxy-mode is only valid for start; later actions inherit the observer origin");
}
if (observeActionRaw !== "start" && args.includes("--sentinel-cadence")) {
throw new Error("web-probe observe --sentinel-cadence is only valid for start");
}
const requestedBrowserProxyMode = browserProxyModeRaw === undefined ? undefined : parseWebProbeBrowserProxyMode(browserProxyModeRaw);
const selectedOrigin = observeActionRaw === "start"
? resolveNodeWebProbeCliOrigin(spec, requestedOrigin, customUrl, requestedBrowserProxyMode)
@@ -645,6 +649,11 @@ export function parseNodeWebProbeObserveOptions(
throw new Error("owning YAML 未启用 validateWorkbenchTraceReadability");
}
}
const resourcePolicy = spec.webProbe?.resourcePolicy;
if (resourcePolicy === undefined) {
throw new Error(`web-probe resource policy is missing from ${hwlabRuntimeLaneConfigPath} for ${node}/${lane}`);
}
const observerLifecycle = resourcePolicy.observerLifecycle;
return {
action: "observe",
observeAction: observeActionRaw,
@@ -659,11 +668,12 @@ export function parseNodeWebProbeObserveOptions(
viewport: optionValue(args, "--viewport") ?? "1440x900",
browserProxyMode: selectedOrigin.browserProxyMode,
browserProxyModeSource: selectedOrigin.browserProxyModeSource,
memoryGuardMode: args.includes("--sentinel-cadence") ? "sentinel-cadence" : "manual-start",
sampleIntervalMs: positiveIntegerOption(args, "--sample-interval-ms", 5000, 600000),
screenshotIntervalMs: positiveIntegerOption(args, "--screenshot-interval-ms", 300000, 86_400_000),
observerRefreshIntervalMs: positiveIntegerOption(args, "--observer-refresh-interval-ms", 180000, 86_400_000),
maxSamples: positiveIntegerOption(args, "--max-samples", 0, 10_000_000),
maxRunSeconds: positiveIntegerOption(args, "--max-run-seconds", 0, 86_400),
maxSamples: observerLifecycleOption(args, "--max-samples", observerLifecycle.maxSamples),
maxRunSeconds: observerLifecycleOption(args, "--max-run-seconds", observerLifecycle.maxRunSeconds),
commandTimeoutSeconds: positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600),
waitMs: positiveIntegerOption(args, "--wait-ms", 0, 600000),
tailLines: positiveIntegerOption(args, "--tail-lines", 5, 200),
@@ -728,6 +738,17 @@ export function parseNodeWebProbeObserveOptions(
};
}
function observerLifecycleOption(args: string[], name: string, yamlMaximum: number): number {
const raw = optionValue(args, name);
if (raw === undefined) return yamlMaximum;
const value = Number(raw);
if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`);
if (value > yamlMaximum) {
throw new Error(`${name}=${value} exceeds the owning YAML observer lifecycle cap ${yamlMaximum}`);
}
return value;
}
export function parseNodeWebProbeObserveCommandType(value: string): NodeWebProbeObserveCommandType {
if (
value === "login"
@@ -870,11 +891,13 @@ export function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, un
if (options.action === "observe") return runNodeWebProbeObserve(options, spec, secretSpec, material, credential);
if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential);
if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential);
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
if (memoryGuard.status !== "allowed") return memoryGuard;
const probeArgs = nodeWebProbeRunArgs(options, "run");
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
const script = [
"set -eu",
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`,
webProbeGuardedLaunchShell(spec, "manual-start", `${[...webProbeProxy.envAssignments, ...webProbeBrowserEvidenceEnvAssignments(spec), `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`),
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const probe = compactWebProbeResult(parseJsonObject(result.stdout));
@@ -912,8 +935,10 @@ export function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, un
}
export function runNodeWebProbeScreenshot(options: NodeWebProbeScreenshotOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
if (memoryGuard.status !== "allowed") return memoryGuard;
const route = `${options.node}:${spec.workspace}`;
const script = webProbeScreenshotRemoteScript(options);
const script = webProbeGuardedLaunchShell(spec, "manual-start", webProbeScreenshotRemoteScript(options, spec));
const job = runWebProbeRemoteArtifactJob({
route,
localDir: options.localDir,
@@ -1038,7 +1063,7 @@ function compactWebProbeScreenshotCleanup(value: unknown): Record<string, unknow
};
}
function webProbeScreenshotRemoteScript(options: NodeWebProbeScreenshotOptions): string {
function webProbeScreenshotRemoteScript(options: NodeWebProbeScreenshotOptions, spec: HwlabRuntimeLaneSpec): string {
const [widthRaw, heightRaw] = options.viewport.split("x");
return [
"set -eu",
@@ -1048,7 +1073,7 @@ function webProbeScreenshotRemoteScript(options: NodeWebProbeScreenshotOptions):
`export UNIDESK_WEB_PROBE_SCREENSHOT_HEIGHT=${shellQuote(heightRaw ?? "900")}`,
`export UNIDESK_WEB_PROBE_SCREENSHOT_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`,
`export UNIDESK_WEB_PROBE_SCREENSHOT_WAIT_UNTIL=${shellQuote(options.waitUntil)}`,
`export UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE=${shellQuote(options.fullPage ? "1" : "0")}`,
`export UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE=${shellQuote(webProbeScreenshotFullPage(spec, options.fullPage) ? "1" : "0")}`,
`export UNIDESK_WEB_PROBE_SCREENSHOT_SELECTOR=${shellQuote(options.selector ?? "")}`,
"if command -v chromium >/dev/null 2>&1; then",
" export UNIDESK_WEB_PROBE_SCREENSHOT_EXECUTABLE_PATH=$(command -v chromium)",
@@ -1075,7 +1100,7 @@ const width = Number(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_WIDTH || 1440);
const height = Number(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_HEIGHT || 900);
const timeout = Number(process.env.UNIDESK_WEB_PROBE_SCREENSHOT_TIMEOUT_MS || 30000);
const waitUntil = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_WAIT_UNTIL || "networkidle";
const fullPage = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE !== "0";
const fullPage = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_FULL_PAGE === "1";
const selector = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_SELECTOR || "";
const executablePath = process.env.UNIDESK_WEB_PROBE_SCREENSHOT_EXECUTABLE_PATH || "";
@@ -1088,8 +1113,11 @@ const launchOptions = {
args: ["--disable-gpu", "--no-sandbox"],
...(executablePath ? { executablePath } : {}),
};
const browser = await chromium.launch(launchOptions);
const context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
let browser = null;
let context = null;
try {
browser = await chromium.launch(launchOptions);
context = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1, isMobile: width <= 560 });
const page = await context.newPage();
page.on("console", (message) => {
if (consoleMessages.length < 20) consoleMessages.push({ type: message.type(), text: message.text().slice(0, 240) });
@@ -1099,7 +1127,6 @@ page.on("requestfailed", (request) => {
});
let status = null;
try {
const response = await page.goto(url, { timeout, waitUntil });
status = response?.status() ?? null;
await page.waitForTimeout(350);
@@ -1168,8 +1195,8 @@ try {
valuesRedacted: true,
}));
} finally {
await context.close().catch(() => {});
await browser.close().catch(() => {});
if (context) await context.close().catch(() => {});
if (browser) await browser.close().catch(() => {});
}
`;
}
@@ -1214,9 +1241,11 @@ export function runNodeWebProbeAsync(
): Record<string, unknown> {
const startArgs = nodeWebProbeRunArgs(options, "start");
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
const memoryGuard = runWebProbeMemoryGuard(spec, "manual-start");
if (memoryGuard.status !== "allowed") return memoryGuard;
const startScript = [
"set -eu",
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`,
webProbeGuardedLaunchShell(spec, "manual-start", `${[...webProbeProxy.envAssignments, ...webProbeBrowserEvidenceEnvAssignments(spec), `HWLAB_WEB_USER=${shellQuote(material.username ?? secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`, "structured-output"),
].join("\n");
const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55);
const start = parseJsonObject(startResult.stdout);
+1 -1
View File
@@ -2130,7 +2130,7 @@ export function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
name: parseWebProbeScreenshotName(optionValue(args, "--name") ?? `web-probe-${node.toLowerCase()}-${lane}.png`),
timeoutMs,
waitUntil: parseWebProbeScreenshotWaitUntil(optionValue(args, "--wait-until") ?? "networkidle"),
fullPage: !args.includes("--no-full-page"),
fullPage: args.includes("--full-page") && !args.includes("--no-full-page"),
selector: optionValue(args, "--selector") ?? null,
keepRemote: args.includes("--keep-remote"),
waitTimeoutMs,