fix: 收敛 WebProbe 物理内存与资源上限
This commit is contained in:
Codex
@@ -0,0 +1,123 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { rootPath } from "./config";
|
||||
|
||||
const webObserveSource = readFileSync(rootPath("scripts/src/gc-remote-web-observe.py"), "utf8");
|
||||
const runnerSource = readFileSync(rootPath("scripts/src/gc-remote-runner.py"), "utf8");
|
||||
|
||||
function runPythonFixture(body: string): Record<string, unknown> {
|
||||
const fixture = [
|
||||
"import json, os, re, signal, time",
|
||||
"PROVIDER_ID = 'NC01'",
|
||||
"def safe_int(value, fallback=0):",
|
||||
" try: return int(value)",
|
||||
" except (TypeError, ValueError): return fallback",
|
||||
"def config_list(config, key, fallback):",
|
||||
" value = config.get(key, fallback)",
|
||||
" return value if isinstance(value, list) else fallback",
|
||||
"def config_int(config, key, fallback, minimum=None, maximum=None):",
|
||||
" value = config.get(key, fallback)",
|
||||
" return value if isinstance(value, int) and not isinstance(value, bool) else fallback",
|
||||
"def config_float(config, key, fallback, minimum=None, maximum=None): return float(config.get(key, fallback))",
|
||||
"def fmt_bytes(value): return str(value)",
|
||||
"def redact_command_preview(value): return str(value)[:200]",
|
||||
"def observe_run_record(path, stale_hours): return {}",
|
||||
webObserveSource,
|
||||
body,
|
||||
].join("\n");
|
||||
const result = spawnSync("python3", ["-"], { input: fixture, encoding: "utf8" });
|
||||
assert.equal(result.status, 0, result.stderr || result.stdout);
|
||||
return JSON.parse(result.stdout.trim()) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
test("memory-pressure candidate fixtures protect active observers and classify only closed stale/orphan trees", () => {
|
||||
const result = runPythonFixture(String.raw`
|
||||
MEMORY_CONFIG = {
|
||||
"observeStateRoots": ["/tmp/observe"], "minimumReadableObserveRoots": 1,
|
||||
"observeRunDepth": 6, "observeScanLimit": 100, "staleRunMaxAgeHours": 1,
|
||||
"processScanLimit": 100, "processStaleMinAgeSeconds": 3600,
|
||||
"maxMemoryReclaimCandidates": 20, "terminationGraceSeconds": 1,
|
||||
"terminationPollMilliseconds": 50, "observerRootPatterns": ["observer-runner.mjs"],
|
||||
"browserRootPatterns": ["observer-runner.mjs", "cliDaemon.mjs"],
|
||||
"browserProcessPatterns": ["chromium", "/usr/bin/chromium"]
|
||||
}
|
||||
def row(pid, ppid, sid, start, age, command, rss=1024):
|
||||
return {"pid": pid, "ppid": ppid, "sid": sid, "startTicks": start, "ageSeconds": age, "rssBytes": rss, "commandLine": command, "commandPreview": command}
|
||||
processes = {
|
||||
100: row(100, 1, 100, 1000, 7200, "node observer-runner.mjs"),
|
||||
101: row(101, 100, 100, 1001, 7200, "/usr/bin/chromium --type=browser"),
|
||||
200: row(200, 1, 200, 2000, 7200, "node observer-runner.mjs"),
|
||||
300: row(300, 1, 300, 3000, 7200, "node cliDaemon.mjs"),
|
||||
301: row(301, 300, 300, 3001, 7200, "/usr/bin/chromium --type=browser"),
|
||||
}
|
||||
read_proc_process_table = lambda: processes
|
||||
observer_run_records = lambda: [
|
||||
{"pid": 100, "pidAlive": True, "staleLiveSignal": False},
|
||||
{"pid": 200, "pidAlive": True, "staleLiveSignal": True},
|
||||
]
|
||||
candidates = collect_memory_reclaim_candidates()
|
||||
print(json.dumps({"kinds": sorted(item["kind"] for item in candidates), "roots": sorted(item["rootPid"] for item in candidates), "diagnostics": memory_reclaim_scan_diagnostics()}))
|
||||
`);
|
||||
assert.deepEqual(result.kinds, ["browser-orphan-process-tree", "stale-web-observer-process-tree"]);
|
||||
assert.deepEqual(result.roots, [200, 300]);
|
||||
assert.equal((result.diagnostics as Record<string, unknown>).blocked, false);
|
||||
});
|
||||
|
||||
test("memory-pressure process scans fail closed on empty or truncated process tables", () => {
|
||||
const result = runPythonFixture(String.raw`
|
||||
MEMORY_CONFIG = {
|
||||
"observeStateRoots": ["/tmp/observe"], "minimumReadableObserveRoots": 1,
|
||||
"observeRunDepth": 6, "observeScanLimit": 100, "staleRunMaxAgeHours": 1,
|
||||
"processScanLimit": 100, "processStaleMinAgeSeconds": 3600,
|
||||
"maxMemoryReclaimCandidates": 20, "terminationGraceSeconds": 1,
|
||||
"terminationPollMilliseconds": 50, "observerRootPatterns": ["observer-runner.mjs"],
|
||||
"browserRootPatterns": ["cliDaemon.mjs"], "browserProcessPatterns": ["chromium"]
|
||||
}
|
||||
observer_run_records = lambda: []
|
||||
read_proc_process_table = lambda: {}
|
||||
empty = collect_memory_reclaim_candidates()
|
||||
empty_diag = memory_reclaim_scan_diagnostics()
|
||||
def truncated_scan():
|
||||
global _PROCESS_SCAN_TRUNCATED
|
||||
_PROCESS_SCAN_TRUNCATED = True
|
||||
return {1: {"pid": 1, "ppid": 0, "sid": 1, "startTicks": 1, "ageSeconds": 7200, "rssBytes": 1, "commandLine": "node cliDaemon.mjs", "commandPreview": "node cliDaemon.mjs"}}
|
||||
read_proc_process_table = truncated_scan
|
||||
truncated = collect_memory_reclaim_candidates()
|
||||
truncated_diag = memory_reclaim_scan_diagnostics()
|
||||
print(json.dumps({"emptyCount": len(empty), "emptyBlocked": empty_diag["blocked"], "emptyError": empty_diag["processScanError"], "truncatedCount": len(truncated), "truncatedBlocked": truncated_diag["blocked"], "truncatedFlag": truncated_diag["processScanTruncated"]}))
|
||||
`);
|
||||
assert.equal(result.emptyCount, 0);
|
||||
assert.equal(result.emptyBlocked, true);
|
||||
assert.equal(result.emptyError, "proc-process-table-empty");
|
||||
assert.equal(result.truncatedCount, 0);
|
||||
assert.equal(result.truncatedBlocked, true);
|
||||
assert.equal(result.truncatedFlag, true);
|
||||
});
|
||||
|
||||
test("post-TERM identity matching tolerates reparenting but never PID startTicks reuse", () => {
|
||||
const result = runPythonFixture(String.raw`
|
||||
identity = {"pid": 41, "ppid": 10, "sid": 10, "startTicks": 1234}
|
||||
reparented = {41: {"pid": 41, "ppid": 1, "sid": 41, "startTicks": 1234}}
|
||||
reused = {41: {"pid": 41, "ppid": 10, "sid": 10, "startTicks": 9999}}
|
||||
print(json.dumps({
|
||||
"preTermReparented": matching_live_identity(identity, reparented) is not None,
|
||||
"postTermReparented": matching_surviving_identity(identity, reparented) is not None,
|
||||
"postTermReused": matching_surviving_identity(identity, reused) is not None,
|
||||
}))
|
||||
`);
|
||||
assert.equal(result.preTermReparented, false);
|
||||
assert.equal(result.postTermReparented, true);
|
||||
assert.equal(result.postTermReused, false);
|
||||
});
|
||||
|
||||
test("memory-pressure async runner uses a parent-child barrier and locked worker state", () => {
|
||||
assert.match(runnerSource, /os\.pipe\(\)/u);
|
||||
assert.match(runnerSource, /os\.read\(read_descriptor/u);
|
||||
assert.match(runnerSource, /workerStartTicks/u);
|
||||
assert.match(runnerSource, /write_memory_job_state\(/u);
|
||||
assert.match(runnerSource, /workerAlive/u);
|
||||
assert.match(runnerSource, /worker-identity-unavailable|worker-exited-without-terminal-state/u);
|
||||
});
|
||||
+357
-10
@@ -1,8 +1,10 @@
|
||||
import base64
|
||||
import calendar
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -207,8 +209,41 @@ def job_paths(job_id):
|
||||
return {
|
||||
"state": os.path.join(REMOTE_GC_JOB_DIR, "%s.json" % job_id),
|
||||
"log": os.path.join(REMOTE_GC_JOB_DIR, "%s.log" % job_id),
|
||||
"lock": os.path.join(REMOTE_GC_JOB_DIR, "%s.lock" % job_id),
|
||||
}
|
||||
|
||||
def process_identity(pid):
|
||||
try:
|
||||
with open("/proc/%s/stat" % int(pid), "r", encoding="utf-8") as handle:
|
||||
text = handle.read().strip()
|
||||
close = text.rfind(")")
|
||||
fields = text[close + 2:].split() if close >= 0 else []
|
||||
return {"state": fields[0], "startTicks": int(fields[19])} if len(fields) >= 20 else None
|
||||
except (OSError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
def process_start_ticks(pid):
|
||||
identity = process_identity(pid)
|
||||
return identity.get("startTicks") if identity else None
|
||||
|
||||
def acquire_job_lock(paths):
|
||||
descriptor = os.open(paths["lock"], os.O_CREAT | os.O_RDWR, 0o600)
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX)
|
||||
return descriptor
|
||||
|
||||
def release_job_lock(descriptor):
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
def write_memory_job_state(paths, payload):
|
||||
descriptor = acquire_job_lock(paths)
|
||||
try:
|
||||
write_json_atomic(paths["state"], payload)
|
||||
finally:
|
||||
release_job_lock(descriptor)
|
||||
|
||||
def status_command(job_id):
|
||||
return "bun scripts/cli.ts gc remote %s status --job-id %s" % (PROVIDER_ID, job_id)
|
||||
|
||||
@@ -326,9 +361,31 @@ def remote_gc_job_status():
|
||||
"statePath": paths["state"],
|
||||
"logTail": read_file_tail(paths["log"]),
|
||||
}
|
||||
descriptor = None
|
||||
try:
|
||||
descriptor = acquire_job_lock(paths)
|
||||
with open(paths["state"], "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if payload.get("scope") == "memory-pressure-only" and payload.get("status") == "running":
|
||||
worker_pid = safe_int(payload.get("workerPid"))
|
||||
expected_start_ticks = safe_int(payload.get("workerStartTicks"))
|
||||
actual_identity = process_identity(worker_pid) if worker_pid > 0 else None
|
||||
actual_start_ticks = actual_identity.get("startTicks") if actual_identity else None
|
||||
worker_alive = actual_identity is not None and actual_identity.get("state") not in set(["Z", "X"]) \
|
||||
and actual_start_ticks == expected_start_ticks
|
||||
if not worker_alive:
|
||||
payload.update({
|
||||
"ok": False,
|
||||
"status": "failed",
|
||||
"outcome": "memory-reclaim-worker-exited-without-terminal-state",
|
||||
"workerAlive": False,
|
||||
"workerObservedState": actual_identity.get("state") if actual_identity else None,
|
||||
"workerObservedStartTicks": actual_start_ticks,
|
||||
"finishedAt": now_iso(),
|
||||
})
|
||||
write_json_atomic(paths["state"], payload)
|
||||
else:
|
||||
payload["workerAlive"] = True
|
||||
except Exception as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
@@ -338,6 +395,9 @@ def remote_gc_job_status():
|
||||
"statePath": paths["state"],
|
||||
"logTail": read_file_tail(paths["log"]),
|
||||
}
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
release_job_lock(descriptor)
|
||||
payload["logTail"] = read_file_tail(paths["log"])
|
||||
return payload
|
||||
|
||||
@@ -634,6 +694,14 @@ def observe_run_record(path, stale_hours):
|
||||
pass
|
||||
if pid is not None:
|
||||
break
|
||||
live_freshness_timestamp = None
|
||||
for source in [heartbeat, manifest]:
|
||||
for key in ["updatedAt", "completedAt", "finishedAt", "stoppedAt"]:
|
||||
live_freshness_timestamp = iso_or_epoch_to_epoch(source.get(key))
|
||||
if live_freshness_timestamp is not None:
|
||||
break
|
||||
if live_freshness_timestamp is not None:
|
||||
break
|
||||
timestamp = None
|
||||
for source in [heartbeat, manifest]:
|
||||
for key in ["updatedAt", "completedAt", "finishedAt", "stoppedAt", "startedAt", "createdAt"]:
|
||||
@@ -648,7 +716,8 @@ def observe_run_record(path, stale_hours):
|
||||
status = heartbeat.get("status") or manifest.get("status") or manifest.get("state")
|
||||
alive = pid_alive(pid)
|
||||
terminal = str(status or "").lower() in set(["done", "completed", "complete", "failed", "blocked", "timeout", "timed-out", "stopped", "exited"])
|
||||
stale_signal = (not alive) and age_hours >= float(stale_hours) and (terminal or status is None)
|
||||
age_stale = age_hours >= float(stale_hours)
|
||||
stale_signal = (not alive) and age_stale and (terminal or status is None)
|
||||
return {
|
||||
"id": os.path.basename(path),
|
||||
"path": path,
|
||||
@@ -658,6 +727,9 @@ def observe_run_record(path, stale_hours):
|
||||
"ageHours": round(age_hours, 2),
|
||||
"timestampBasis": "manifest-or-heartbeat" if heartbeat or manifest else "directory-mtime-fallback",
|
||||
"staleSignal": stale_signal,
|
||||
"staleLiveSignal": alive and age_stale and live_freshness_timestamp is not None,
|
||||
"activeSignal": alive and not (age_stale and live_freshness_timestamp is not None),
|
||||
"liveStaleEvidence": "heartbeat-or-manifest-update" if live_freshness_timestamp is not None else None,
|
||||
"classification": "review-only",
|
||||
}
|
||||
|
||||
@@ -673,6 +745,7 @@ def collect_web_observe_summary():
|
||||
}
|
||||
root_rows = []
|
||||
stale_rows = []
|
||||
stale_live_rows = []
|
||||
active_rows = []
|
||||
run_count = 0
|
||||
total_bytes = 0
|
||||
@@ -691,22 +764,18 @@ def collect_web_observe_summary():
|
||||
"activeSignalCount": 0,
|
||||
}
|
||||
if exists:
|
||||
try:
|
||||
children = [os.path.join(root, name) for name in os.listdir(root)]
|
||||
except OSError:
|
||||
children = []
|
||||
for child in children:
|
||||
if not os.path.isdir(child):
|
||||
continue
|
||||
for child in configured_observe_runs(root):
|
||||
try:
|
||||
record = observe_run_record(child, stale_hours)
|
||||
except OSError:
|
||||
continue
|
||||
row["runCount"] += 1
|
||||
run_count += 1
|
||||
if record.get("pidAlive"):
|
||||
if record.get("activeSignal"):
|
||||
row["activeSignalCount"] += 1
|
||||
active_rows.append(record)
|
||||
if record.get("staleLiveSignal"):
|
||||
stale_live_rows.append(record)
|
||||
if record.get("staleSignal"):
|
||||
row["staleSignalCount"] += 1
|
||||
stale_rows.append(record)
|
||||
@@ -723,9 +792,11 @@ def collect_web_observe_summary():
|
||||
"runCount": run_count,
|
||||
"activeSignalCount": len(active_rows),
|
||||
"staleSignalCount": len(stale_rows),
|
||||
"staleLiveSignalCount": len(stale_live_rows),
|
||||
"roots": root_rows,
|
||||
"activeSignals": active_rows[:int(OPTIONS.get("limit") or 50)],
|
||||
"staleSignals": stale_rows[:int(OPTIONS.get("limit") or 50)],
|
||||
"staleLiveSignals": stale_live_rows[:int(OPTIONS.get("limit") or 50)],
|
||||
"policy": "analysis-only; active or stale observe runs must be stopped/retained through controlled observer lifecycle commands, not raw process kill or directory deletion",
|
||||
}
|
||||
|
||||
@@ -993,6 +1064,8 @@ def collect_protected():
|
||||
return result
|
||||
|
||||
def collect_candidates(observed_at):
|
||||
if bool(OPTIONS.get("memoryPressureOnly")):
|
||||
return collect_memory_reclaim_candidates()
|
||||
candidates = []
|
||||
if OPTIONS.get("journal", True):
|
||||
usage = command(["journalctl", "--disk-usage"], 5)
|
||||
@@ -1340,6 +1413,29 @@ def summarize(candidates, returned, disk=None):
|
||||
"target": target_assessment(disk, total),
|
||||
}
|
||||
|
||||
def summarize_memory_candidates(candidates, returned):
|
||||
total_memory = sum(safe_int(item.get("expectedMemoryRssBytes")) for item in candidates)
|
||||
returned_memory = sum(safe_int(item.get("expectedMemoryRssBytes")) for item in returned)
|
||||
by_kind = {}
|
||||
for item in candidates:
|
||||
kind = item.get("kind") or "unknown"
|
||||
bucket = by_kind.setdefault(kind, {"count": 0, "expectedMemoryRssBytes": 0})
|
||||
bucket["count"] += 1
|
||||
bucket["expectedMemoryRssBytes"] += safe_int(item.get("expectedMemoryRssBytes"))
|
||||
bucket["expectedMemoryRss"] = fmt_bytes(bucket["expectedMemoryRssBytes"])
|
||||
return {
|
||||
"scope": "memory-pressure-only",
|
||||
"candidateCount": len(candidates),
|
||||
"returnedCandidateCount": len(returned),
|
||||
"expectedMemoryRssBytes": total_memory,
|
||||
"expectedMemoryRss": fmt_bytes(total_memory),
|
||||
"returnedExpectedMemoryRssBytes": returned_memory,
|
||||
"returnedExpectedMemoryRss": fmt_bytes(returned_memory),
|
||||
"estimatedDiskBytes": 0,
|
||||
"outcome": "memory-reclaim-candidates" if candidates else "no-memory-reclaim",
|
||||
"byKind": by_kind,
|
||||
}
|
||||
|
||||
def assert_tmp_candidate(path):
|
||||
resolved = os.path.abspath(path)
|
||||
if not resolved.startswith("/tmp/"):
|
||||
@@ -1426,6 +1522,8 @@ def execute(candidate):
|
||||
before = du_size(path, 15) or path_size(path)
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
return {"reclaimedBytes": before, "stale": record}
|
||||
if kind in set(["browser-orphan-process-tree", "stale-web-observer-process-tree"]):
|
||||
return execute_memory_process_candidate(candidate)
|
||||
if kind == "k3s-cri-image-prune":
|
||||
return execute_k3s_image_cache_prune()
|
||||
if kind == "host-containerd-cache-prune":
|
||||
@@ -1476,6 +1574,13 @@ def returned_results(results):
|
||||
"path",
|
||||
"status",
|
||||
"estimatedReclaimBytes",
|
||||
"estimatedDiskBytes",
|
||||
"expectedMemoryRssBytes",
|
||||
"reclaimedMemoryBytes",
|
||||
"processCount",
|
||||
"termSignalCount",
|
||||
"killSignalCount",
|
||||
"remainingIdentityCount",
|
||||
"reclaimedBytes",
|
||||
"orphanCount",
|
||||
"selectedOrphanCount",
|
||||
@@ -1494,6 +1599,38 @@ def returned_results(results):
|
||||
return [compact_result(item) for item in (failed + started + succeeded)[:int(OPTIONS.get("resultLimit") or 50)]]
|
||||
|
||||
def plan_payload(observed_at, preflight, protected, candidates, visible):
|
||||
if bool(OPTIONS.get("memoryPressureOnly")):
|
||||
scan = memory_reclaim_scan_diagnostics()
|
||||
summary = summarize_memory_candidates(candidates, visible)
|
||||
if scan.get("blocked"):
|
||||
summary["outcome"] = "memory-reclaim-scan-blocked"
|
||||
return {
|
||||
"ok": scan.get("ok") is True,
|
||||
"action": "gc remote plan",
|
||||
"providerId": PROVIDER_ID,
|
||||
"scope": "memory-pressure-only",
|
||||
"dryRun": True,
|
||||
"mutation": False,
|
||||
"observedAt": observed_at,
|
||||
"options": OPTIONS,
|
||||
"memoryBefore": collect_memory_snapshot(),
|
||||
"summary": summary,
|
||||
"candidateScan": scan,
|
||||
"candidates": visible,
|
||||
"protected": {
|
||||
"activeObserverIntersection": "excluded" if scan.get("ok") is True else "unknown-protected-all",
|
||||
"processIdentity": "pre-term:pid-ppid-sid-startTicks;post-term:pid-startTicks",
|
||||
"treeClosure": "unique-root-ppid-sid-closed",
|
||||
},
|
||||
"policy": {
|
||||
"requiresRunConfirm": True,
|
||||
"planCommand": "bun scripts/cli.ts gc remote %s plan --memory-pressure-only" % PROVIDER_ID,
|
||||
"runCommand": "bun scripts/cli.ts gc remote %s run --confirm --memory-pressure-only" % PROVIDER_ID,
|
||||
"candidateKinds": ["browser-orphan-process-tree", "stale-web-observer-process-tree"],
|
||||
"neverTouches": ["disk candidates", "apt cache", "journal", "tmp", "container images", "active observer process trees"],
|
||||
"termination": "SIGTERM, bounded wait, identity reclose, SIGKILL only same-tree residual",
|
||||
},
|
||||
}
|
||||
disk = df_snapshot()
|
||||
ci_storage = ci_storage_snapshot()
|
||||
memory_pressure = collect_memory_pressure()
|
||||
@@ -1819,6 +1956,205 @@ def remote_policy_install_payload(observed_at):
|
||||
},
|
||||
}
|
||||
|
||||
def update_memory_pressure_job_progress(paths, completed_count, candidate_count, worker_pid, worker_start_ticks):
|
||||
descriptor = acquire_job_lock(paths)
|
||||
try:
|
||||
with open(paths["state"], "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if payload.get("status") != "running" or safe_int(payload.get("workerPid")) != worker_pid \
|
||||
or safe_int(payload.get("workerStartTicks")) != worker_start_ticks:
|
||||
raise RuntimeError("memory pressure job state no longer belongs to this worker")
|
||||
payload.update({
|
||||
"workerHeartbeatAt": now_iso(),
|
||||
"progress": {"completedCount": completed_count, "candidateCount": candidate_count},
|
||||
})
|
||||
write_json_atomic(paths["state"], payload)
|
||||
finally:
|
||||
release_job_lock(descriptor)
|
||||
|
||||
def finish_memory_pressure_job(job_id, candidates, observed_at, worker_pid, worker_start_ticks):
|
||||
paths = job_paths(job_id)
|
||||
memory_before = collect_memory_snapshot()
|
||||
results = []
|
||||
update_memory_pressure_job_progress(paths, 0, len(candidates), worker_pid, worker_start_ticks)
|
||||
for index, candidate in enumerate(candidates):
|
||||
try:
|
||||
execution = execute(candidate)
|
||||
item = dict(candidate)
|
||||
item.update({
|
||||
"status": execution.get("status") or "succeeded",
|
||||
"reclaimedBytes": 0,
|
||||
"reclaimedMemoryBytes": execution.get("reclaimedMemoryBytes"),
|
||||
"termSignalCount": execution.get("termSignalCount"),
|
||||
"killSignalCount": execution.get("killSignalCount"),
|
||||
"remainingIdentityCount": execution.get("remainingIdentityCount"),
|
||||
"remainingIdentities": execution.get("remainingIdentities"),
|
||||
"lateAllowlistedIdentityCount": execution.get("lateAllowlistedIdentityCount"),
|
||||
"memoryBefore": execution.get("memoryBefore"),
|
||||
"memoryAfter": execution.get("memoryAfter"),
|
||||
})
|
||||
results.append(item)
|
||||
except Exception as exc:
|
||||
item = dict(candidate)
|
||||
item.update({"status": "failed", "reclaimedBytes": 0, "reclaimedMemoryBytes": None, "error": str(exc)})
|
||||
results.append(item)
|
||||
update_memory_pressure_job_progress(paths, index + 1, len(candidates), worker_pid, worker_start_ticks)
|
||||
memory_after = collect_memory_snapshot()
|
||||
failed = [item for item in results if item.get("status") == "failed"]
|
||||
returned = returned_results(results)
|
||||
summary = summarize_memory_candidates(candidates, returned)
|
||||
summary.update({
|
||||
"attemptedCount": len(results),
|
||||
"succeededCount": len([item for item in results if item.get("status") == "succeeded"]),
|
||||
"failedCount": len(failed),
|
||||
"actualMemoryAvailableDeltaBytes": (
|
||||
safe_int((memory_after.get("memory") or {}).get("availableBytes"))
|
||||
- safe_int((memory_before.get("memory") or {}).get("availableBytes"))
|
||||
) if memory_before.get("ok") and memory_after.get("ok") else None,
|
||||
})
|
||||
payload = {
|
||||
"ok": len(failed) == 0,
|
||||
"action": "gc remote run",
|
||||
"providerId": PROVIDER_ID,
|
||||
"scope": "memory-pressure-only",
|
||||
"status": "succeeded" if len(failed) == 0 else "failed",
|
||||
"jobId": job_id,
|
||||
"statusCommand": status_command(job_id),
|
||||
"statePath": paths["state"],
|
||||
"dryRun": False,
|
||||
"mutation": len(results) > 0,
|
||||
"observedAt": observed_at,
|
||||
"startedAt": observed_at,
|
||||
"finishedAt": now_iso(),
|
||||
"workerPid": worker_pid,
|
||||
"workerStartTicks": worker_start_ticks,
|
||||
"workerAlive": False,
|
||||
"workerCompleted": True,
|
||||
"workerHeartbeatAt": now_iso(),
|
||||
"progress": {"completedCount": len(candidates), "candidateCount": len(candidates)},
|
||||
"memoryBefore": memory_before,
|
||||
"memoryAfter": memory_after,
|
||||
"summary": summary,
|
||||
"results": returned,
|
||||
}
|
||||
write_memory_job_state(paths, payload)
|
||||
|
||||
def start_memory_pressure_job(observed_at, candidates):
|
||||
job_id = "%s-memory-pressure-%s-%s" % (
|
||||
re.sub(r"[^A-Za-z0-9._-]+", "-", PROVIDER_ID.lower()).strip("-") or "provider",
|
||||
int(time.time()),
|
||||
os.getpid(),
|
||||
)
|
||||
paths = job_paths(job_id)
|
||||
summary = summarize_memory_candidates(candidates, candidates)
|
||||
scan = memory_reclaim_scan_diagnostics()
|
||||
base = {
|
||||
"ok": scan.get("ok") is True,
|
||||
"action": "gc remote run",
|
||||
"providerId": PROVIDER_ID,
|
||||
"scope": "memory-pressure-only",
|
||||
"jobId": job_id,
|
||||
"statusCommand": status_command(job_id),
|
||||
"statePath": paths["state"],
|
||||
"dryRun": False,
|
||||
"mutation": len(candidates) > 0,
|
||||
"observedAt": observed_at,
|
||||
"startedAt": observed_at,
|
||||
"summary": summary,
|
||||
"candidateScan": scan,
|
||||
}
|
||||
if scan.get("blocked"):
|
||||
terminal = dict(base)
|
||||
terminal.update({
|
||||
"status": "blocked",
|
||||
"outcome": "memory-reclaim-scan-blocked",
|
||||
"mutation": False,
|
||||
"finishedAt": now_iso(),
|
||||
"results": [],
|
||||
})
|
||||
write_json_atomic(paths["state"], terminal)
|
||||
return terminal
|
||||
if not candidates:
|
||||
terminal = dict(base)
|
||||
terminal.update({
|
||||
"status": "blocked",
|
||||
"outcome": "no-memory-reclaim-candidates",
|
||||
"mutation": False,
|
||||
"finishedAt": now_iso(),
|
||||
"results": [],
|
||||
})
|
||||
write_json_atomic(paths["state"], terminal)
|
||||
return terminal
|
||||
read_descriptor, write_descriptor = os.pipe()
|
||||
try:
|
||||
child_pid = os.fork()
|
||||
except OSError as exc:
|
||||
os.close(read_descriptor)
|
||||
os.close(write_descriptor)
|
||||
failed = dict(base)
|
||||
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-fork-failed", "error": str(exc), "finishedAt": now_iso()})
|
||||
write_memory_job_state(paths, failed)
|
||||
return failed
|
||||
if child_pid == 0:
|
||||
try:
|
||||
os.close(write_descriptor)
|
||||
released = os.read(read_descriptor, 1)
|
||||
os.close(read_descriptor)
|
||||
if released != b"1":
|
||||
os._exit(1)
|
||||
os.setsid()
|
||||
with open("/dev/null", "rb", buffering=0) as stdin_handle, open(paths["log"], "ab", buffering=0) as log_handle:
|
||||
os.dup2(stdin_handle.fileno(), 0)
|
||||
os.dup2(log_handle.fileno(), 1)
|
||||
os.dup2(log_handle.fileno(), 2)
|
||||
finish_memory_pressure_job(job_id, candidates, observed_at, os.getpid(), process_start_ticks(os.getpid()))
|
||||
except Exception as exc:
|
||||
failed = dict(base)
|
||||
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-worker-failed", "workerPid": os.getpid(), "workerStartTicks": process_start_ticks(os.getpid()), "error": str(exc), "finishedAt": now_iso()})
|
||||
try:
|
||||
write_memory_job_state(paths, failed)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
os._exit(0)
|
||||
os.close(read_descriptor)
|
||||
worker_start_ticks = process_start_ticks(child_pid)
|
||||
if worker_start_ticks is None:
|
||||
os.close(write_descriptor)
|
||||
try:
|
||||
os.kill(child_pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
failed = dict(base)
|
||||
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-worker-identity-unavailable", "workerPid": child_pid, "finishedAt": now_iso()})
|
||||
write_memory_job_state(paths, failed)
|
||||
return failed
|
||||
running = dict(base)
|
||||
running.update({
|
||||
"status": "running",
|
||||
"outcome": "memory-reclaim-started",
|
||||
"workerPid": child_pid,
|
||||
"workerStartTicks": worker_start_ticks,
|
||||
"workerAlive": True,
|
||||
"workerHeartbeatAt": now_iso(),
|
||||
"progress": {"completedCount": 0, "candidateCount": len(candidates)},
|
||||
})
|
||||
try:
|
||||
write_memory_job_state(paths, running)
|
||||
os.write(write_descriptor, b"1")
|
||||
except Exception as exc:
|
||||
try:
|
||||
os.kill(child_pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
failed = dict(base)
|
||||
failed.update({"ok": False, "status": "failed", "outcome": "memory-reclaim-worker-release-failed", "workerPid": child_pid, "workerStartTicks": worker_start_ticks, "error": str(exc), "finishedAt": now_iso()})
|
||||
write_memory_job_state(paths, failed)
|
||||
return failed
|
||||
finally:
|
||||
os.close(write_descriptor)
|
||||
return running
|
||||
|
||||
def main():
|
||||
observed_at = now_iso()
|
||||
if ACTION == "memory":
|
||||
@@ -1838,6 +2174,17 @@ def main():
|
||||
"valuesRedacted": True,
|
||||
}, persist_large=False)
|
||||
return 0
|
||||
if ACTION == "status" and job_id_or_none():
|
||||
emit_json(remote_gc_job_status(), persist_large=False)
|
||||
return 0
|
||||
if bool(OPTIONS.get("memoryPressureOnly")) and ACTION in set(["plan", "run"]):
|
||||
candidates = collect_memory_reclaim_candidates()
|
||||
visible = visible_items(candidates)
|
||||
if ACTION == "plan":
|
||||
emit_json(plan_payload(observed_at, {"skipped": True, "reason": "memory-pressure-only"}, [], candidates, visible), persist_large=True)
|
||||
else:
|
||||
emit_json(start_memory_pressure_job(observed_at, visible), persist_large=True)
|
||||
return 0
|
||||
preflight = cluster_preflight()
|
||||
if ACTION == "policy-plan":
|
||||
emit_json(remote_policy_plan_payload(observed_at), persist_large=False)
|
||||
@@ -1897,7 +2244,7 @@ def main():
|
||||
})
|
||||
emit_json(snapshot, persist_large=True)
|
||||
return 0
|
||||
protected = collect_protected()
|
||||
protected = [] if bool(OPTIONS.get("memoryPressureOnly")) else collect_protected()
|
||||
candidates = collect_candidates(observed_at)
|
||||
visible = visible_items(candidates)
|
||||
if ACTION == "plan":
|
||||
|
||||
@@ -1,14 +1,63 @@
|
||||
_OBSERVE_RUN_CACHE = {}
|
||||
_OBSERVE_SCAN_TRUNCATED = False
|
||||
_PROCESS_SCAN_TRUNCATED = False
|
||||
_OBSERVE_SCAN_ERROR = None
|
||||
_PROCESS_SCAN_ERROR = None
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = None
|
||||
_OBSERVE_ROOT_DIAGNOSTICS = {"configured": [], "readable": [], "missing": []}
|
||||
|
||||
def configured_observe_roots():
|
||||
roots = config_list(MEMORY_CONFIG, "observeStateRoots", config_list(MEMORY_CONFIG, "webObserveRoots", []))
|
||||
return [os.path.abspath(item) for item in roots if isinstance(item, str) and item.startswith("/")]
|
||||
|
||||
def is_direct_observe_run_path(path):
|
||||
def configured_observe_runs(root):
|
||||
global _OBSERVE_SCAN_TRUNCATED, _OBSERVE_SCAN_ERROR
|
||||
resolved_root = os.path.abspath(root)
|
||||
if resolved_root in _OBSERVE_RUN_CACHE:
|
||||
return list(_OBSERVE_RUN_CACHE[resolved_root])
|
||||
max_depth = config_int(MEMORY_CONFIG, "observeRunDepth", 1, minimum=1, maximum=12)
|
||||
scan_limit = config_int(MEMORY_CONFIG, "observeScanLimit", 1000, minimum=1, maximum=100000)
|
||||
markers = set(["manifest.json", "heartbeat.json", "pid", "observer.pid", "runner.pid"])
|
||||
runs = []
|
||||
scanned = 0
|
||||
if not os.path.isdir(resolved_root) or os.path.islink(resolved_root):
|
||||
return runs
|
||||
def record_walk_error(error):
|
||||
global _OBSERVE_SCAN_ERROR
|
||||
_OBSERVE_SCAN_ERROR = "observe-state-walk-failed:%s" % type(error).__name__
|
||||
|
||||
for current, directories, files in os.walk(resolved_root, topdown=True, followlinks=False, onerror=record_walk_error):
|
||||
relative = os.path.relpath(current, resolved_root)
|
||||
depth = 0 if relative == "." else relative.count(os.sep) + 1
|
||||
directories[:] = sorted(name for name in directories if not os.path.islink(os.path.join(current, name)))
|
||||
if depth >= max_depth:
|
||||
directories[:] = []
|
||||
if depth == 0:
|
||||
continue
|
||||
scanned += 1
|
||||
if scanned > scan_limit:
|
||||
_OBSERVE_SCAN_TRUNCATED = True
|
||||
break
|
||||
if markers.intersection(files):
|
||||
runs.append(os.path.abspath(current))
|
||||
directories[:] = []
|
||||
if not _OBSERVE_SCAN_TRUNCATED:
|
||||
_OBSERVE_RUN_CACHE[resolved_root] = list(runs)
|
||||
return runs
|
||||
|
||||
def is_observe_run_path(path):
|
||||
resolved = os.path.abspath(path)
|
||||
for root in configured_observe_roots():
|
||||
if os.path.dirname(resolved) == root and resolved.startswith(root.rstrip("/") + "/"):
|
||||
prefix = root.rstrip("/") + "/"
|
||||
if resolved.startswith(prefix) and resolved in set(configured_observe_runs(root)):
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_direct_observe_run_path(path):
|
||||
# Kept as a compatibility name for generic artifact retention. Validation is
|
||||
# now marker/depth based because the configured roots contain dated trees.
|
||||
return is_observe_run_path(path)
|
||||
|
||||
def path_has_open_fd(path):
|
||||
resolved = os.path.realpath(path)
|
||||
prefix = resolved.rstrip("/") + "/"
|
||||
@@ -42,8 +91,8 @@ def path_has_open_fd(path):
|
||||
|
||||
def assert_web_observe_candidate(path):
|
||||
resolved = os.path.abspath(path)
|
||||
if not is_direct_observe_run_path(resolved):
|
||||
raise RuntimeError("refusing to remove web-observe path outside configured direct run roots: %s" % path)
|
||||
if not is_observe_run_path(resolved):
|
||||
raise RuntimeError("refusing to remove web-observe path outside configured marker/depth run roots: %s" % path)
|
||||
if os.path.islink(resolved) or not os.path.isdir(resolved):
|
||||
raise RuntimeError("refusing to remove non-directory or symlink web-observe path: %s" % path)
|
||||
stale_hours = config_float(MEMORY_CONFIG, "staleRunMaxAgeHours", 6.0, minimum=0.0)
|
||||
@@ -55,3 +104,423 @@ def assert_web_observe_candidate(path):
|
||||
if path_has_open_fd(resolved):
|
||||
raise RuntimeError("refusing to remove web-observe run with open fd/cwd reference: %s" % path)
|
||||
return record
|
||||
|
||||
def read_proc_process_table():
|
||||
global _PROCESS_SCAN_TRUNCATED, _PROCESS_SCAN_ERROR
|
||||
scan_limit = config_int(MEMORY_CONFIG, "processScanLimit", 16384, minimum=1, maximum=1000000)
|
||||
try:
|
||||
clock_ticks = int(os.sysconf("SC_CLK_TCK"))
|
||||
uptime = float(open("/proc/uptime", "r", encoding="utf-8").read().split()[0])
|
||||
page_size = int(os.sysconf("SC_PAGE_SIZE"))
|
||||
pid_names = sorted((name for name in os.listdir("/proc") if name.isdigit()), key=int)
|
||||
if len(pid_names) > scan_limit:
|
||||
_PROCESS_SCAN_TRUNCATED = True
|
||||
return {}
|
||||
except Exception as exc:
|
||||
_PROCESS_SCAN_ERROR = "proc-process-table-read-failed:%s" % type(exc).__name__
|
||||
return {}
|
||||
rows = {}
|
||||
stat_read_failures = 0
|
||||
for pid_name in pid_names:
|
||||
base = os.path.join("/proc", pid_name)
|
||||
try:
|
||||
with open(os.path.join(base, "stat"), "r", encoding="utf-8") as handle:
|
||||
stat_text = handle.read().strip()
|
||||
close = stat_text.rfind(")")
|
||||
if close < 0:
|
||||
continue
|
||||
fields = stat_text[close + 2:].split()
|
||||
if len(fields) < 22:
|
||||
continue
|
||||
pid = int(pid_name)
|
||||
ppid = int(fields[1])
|
||||
sid = int(fields[3])
|
||||
start_ticks = int(fields[19])
|
||||
try:
|
||||
with open(os.path.join(base, "cmdline"), "rb") as handle:
|
||||
command_line = handle.read(65536).replace(b"\0", b" ").decode("utf-8", errors="replace").strip()
|
||||
except OSError:
|
||||
command_line = ""
|
||||
try:
|
||||
with open(os.path.join(base, "statm"), "r", encoding="utf-8") as handle:
|
||||
rss_bytes = safe_int(handle.read().split()[1]) * page_size
|
||||
except Exception:
|
||||
rss_bytes = 0
|
||||
rows[pid] = {
|
||||
"pid": pid,
|
||||
"ppid": ppid,
|
||||
"sid": sid,
|
||||
"startTicks": start_ticks,
|
||||
"ageSeconds": max(0, int(uptime - (float(start_ticks) / float(clock_ticks)))),
|
||||
"rssBytes": rss_bytes,
|
||||
"commandPreview": redact_command_preview(command_line),
|
||||
"commandLine": command_line,
|
||||
}
|
||||
except (OSError, ValueError, IndexError):
|
||||
stat_read_failures += 1
|
||||
continue
|
||||
if not rows:
|
||||
_PROCESS_SCAN_ERROR = "proc-process-table-empty"
|
||||
elif stat_read_failures > max(32, len(pid_names) // 2):
|
||||
_PROCESS_SCAN_ERROR = "proc-process-stat-read-failure-threshold:%s-of-%s" % (stat_read_failures, len(pid_names))
|
||||
return rows
|
||||
|
||||
def validate_observe_run_markers(path):
|
||||
for name in ["heartbeat.json", "manifest.json"]:
|
||||
marker = os.path.join(path, name)
|
||||
if not os.path.lexists(marker):
|
||||
continue
|
||||
if os.path.islink(marker) or not os.path.isfile(marker):
|
||||
raise OSError("observe marker is not a regular file: %s" % marker)
|
||||
with open(marker, "r", encoding="utf-8") as handle:
|
||||
value = json.load(handle)
|
||||
if not isinstance(value, dict):
|
||||
raise OSError("observe JSON marker is not an object: %s" % marker)
|
||||
for name in ["pid", "observer.pid", "browser.pid", "runner.pid"]:
|
||||
marker = os.path.join(path, name)
|
||||
if not os.path.lexists(marker):
|
||||
continue
|
||||
if os.path.islink(marker) or not os.path.isfile(marker):
|
||||
raise OSError("observe pid marker is not a regular file: %s" % marker)
|
||||
with open(marker, "r", encoding="utf-8") as handle:
|
||||
raw = handle.read().strip()
|
||||
if not re.match(r"^[1-9]\d*$", raw):
|
||||
raise OSError("observe pid marker is invalid: %s" % marker)
|
||||
|
||||
def observer_run_records():
|
||||
global _OBSERVE_SCAN_ERROR, _OBSERVE_ROOT_DIAGNOSTICS
|
||||
stale_hours = config_float(MEMORY_CONFIG, "staleRunMaxAgeHours", 6.0, minimum=0.0)
|
||||
records = []
|
||||
roots = configured_observe_roots()
|
||||
accessible_roots = []
|
||||
missing_roots = []
|
||||
for root in roots:
|
||||
try:
|
||||
root_stat = os.stat(root)
|
||||
if os.path.isdir(root) and not os.path.islink(root):
|
||||
accessible_roots.append(root)
|
||||
elif root_stat:
|
||||
_OBSERVE_SCAN_ERROR = "observe-state-root-not-directory"
|
||||
except FileNotFoundError:
|
||||
missing_roots.append(root)
|
||||
continue
|
||||
except OSError as exc:
|
||||
_OBSERVE_SCAN_ERROR = "observe-state-root-read-failed:%s" % type(exc).__name__
|
||||
minimum_readable = config_int(MEMORY_CONFIG, "minimumReadableObserveRoots", 1, minimum=1, maximum=100)
|
||||
_OBSERVE_ROOT_DIAGNOSTICS = {"configured": list(roots), "readable": list(accessible_roots), "missing": missing_roots}
|
||||
if len(accessible_roots) < minimum_readable:
|
||||
_OBSERVE_SCAN_ERROR = _OBSERVE_SCAN_ERROR or "minimum-readable-observe-roots-not-met:%s-of-%s" % (len(accessible_roots), minimum_readable)
|
||||
for root in accessible_roots:
|
||||
for path in configured_observe_runs(root):
|
||||
try:
|
||||
validate_observe_run_markers(path)
|
||||
records.append(observe_run_record(path, stale_hours))
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
_OBSERVE_SCAN_ERROR = "observe-run-marker-read-failed:%s" % type(exc).__name__
|
||||
continue
|
||||
return records
|
||||
|
||||
def process_descendants(processes, root_pid):
|
||||
selected = set([int(root_pid)])
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for pid, row in processes.items():
|
||||
if pid not in selected and safe_int(row.get("ppid")) in selected:
|
||||
selected.add(pid)
|
||||
changed = True
|
||||
return selected
|
||||
|
||||
def session_closed_tree(processes, sid):
|
||||
members = {pid: row for pid, row in processes.items() if safe_int(row.get("sid")) == int(sid)}
|
||||
if not members:
|
||||
return None
|
||||
roots = [pid for pid, row in members.items() if safe_int(row.get("ppid")) not in members]
|
||||
if len(roots) != 1:
|
||||
return None
|
||||
root_pid = roots[0]
|
||||
if process_descendants(members, root_pid) != set(members.keys()):
|
||||
return None
|
||||
return {"rootPid": root_pid, "members": members}
|
||||
|
||||
def command_matches(row, patterns):
|
||||
haystack = str(row.get("commandLine") or "").lower()
|
||||
return any(str(pattern).lower() in haystack for pattern in patterns if str(pattern))
|
||||
|
||||
def memory_process_candidate_id(kind, root):
|
||||
return "%s:%s:%s:%s" % (kind, safe_int(root.get("sid")), safe_int(root.get("pid")), safe_int(root.get("startTicks")))
|
||||
|
||||
def collect_memory_reclaim_candidates():
|
||||
global _OBSERVE_SCAN_TRUNCATED, _PROCESS_SCAN_TRUNCATED, _OBSERVE_SCAN_ERROR, _PROCESS_SCAN_ERROR, _MEMORY_RECLAIM_CONFIG_ERROR, _OBSERVE_ROOT_DIAGNOSTICS
|
||||
_OBSERVE_SCAN_TRUNCATED = False
|
||||
_PROCESS_SCAN_TRUNCATED = False
|
||||
_OBSERVE_SCAN_ERROR = None
|
||||
_PROCESS_SCAN_ERROR = None
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = None
|
||||
_OBSERVE_ROOT_DIAGNOSTICS = {"configured": [], "readable": [], "missing": []}
|
||||
_OBSERVE_RUN_CACHE.clear()
|
||||
required_keys = [
|
||||
"observeStateRoots", "minimumReadableObserveRoots", "observeRunDepth", "observeScanLimit", "staleRunMaxAgeHours",
|
||||
"processScanLimit", "processStaleMinAgeSeconds", "maxMemoryReclaimCandidates",
|
||||
"terminationGraceSeconds", "terminationPollMilliseconds", "observerRootPatterns",
|
||||
"browserRootPatterns", "browserProcessPatterns",
|
||||
]
|
||||
if any(key not in MEMORY_CONFIG for key in required_keys):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-policy-missing"
|
||||
return []
|
||||
list_keys = ["observeStateRoots", "observerRootPatterns", "browserRootPatterns", "browserProcessPatterns"]
|
||||
integer_keys = [
|
||||
"observeRunDepth", "observeScanLimit", "processScanLimit", "processStaleMinAgeSeconds",
|
||||
"maxMemoryReclaimCandidates", "terminationGraceSeconds", "terminationPollMilliseconds",
|
||||
]
|
||||
if any(not isinstance(MEMORY_CONFIG.get(key), list) or not MEMORY_CONFIG.get(key) for key in list_keys):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-list-invalid"
|
||||
return []
|
||||
if any(any(not isinstance(item, str) or not item.strip() or "\0" in item for item in MEMORY_CONFIG.get(key)) for key in list_keys):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-list-entry-invalid"
|
||||
return []
|
||||
if any(not os.path.isabs(item) for item in MEMORY_CONFIG.get("observeStateRoots")):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-observe-root-not-absolute"
|
||||
return []
|
||||
integer_ranges = {
|
||||
"observeRunDepth": (1, 12),
|
||||
"minimumReadableObserveRoots": (1, 100),
|
||||
"observeScanLimit": (1, 100000),
|
||||
"processScanLimit": (1, 1000000),
|
||||
"processStaleMinAgeSeconds": (60, 604800),
|
||||
"maxMemoryReclaimCandidates": (1, 100),
|
||||
"terminationGraceSeconds": (1, 30),
|
||||
"terminationPollMilliseconds": (50, 1000),
|
||||
}
|
||||
if any(isinstance(MEMORY_CONFIG.get(key), bool) or not isinstance(MEMORY_CONFIG.get(key), int)
|
||||
or MEMORY_CONFIG.get(key) < minimum or MEMORY_CONFIG.get(key) > maximum
|
||||
for key, (minimum, maximum) in integer_ranges.items()):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-memory-pressure-integer-invalid"
|
||||
return []
|
||||
if MEMORY_CONFIG.get("minimumReadableObserveRoots") > len(MEMORY_CONFIG.get("observeStateRoots")):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "minimum-readable-observe-roots-exceeds-configured-roots"
|
||||
return []
|
||||
stale_hours = MEMORY_CONFIG.get("staleRunMaxAgeHours")
|
||||
if isinstance(stale_hours, bool) or not isinstance(stale_hours, (int, float)) or stale_hours <= 0 or stale_hours > 8760:
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "required-yaml-stale-age-invalid"
|
||||
return []
|
||||
browser_patterns = config_list(MEMORY_CONFIG, "browserProcessPatterns", [])
|
||||
root_patterns = config_list(MEMORY_CONFIG, "browserRootPatterns", [])
|
||||
observer_patterns = config_list(MEMORY_CONFIG, "observerRootPatterns", [])
|
||||
stale_seconds = config_int(MEMORY_CONFIG, "processStaleMinAgeSeconds", 3600, minimum=60, maximum=604800)
|
||||
if not browser_patterns or not root_patterns or not observer_patterns:
|
||||
return []
|
||||
processes = read_proc_process_table()
|
||||
records = observer_run_records()
|
||||
if not processes and _PROCESS_SCAN_ERROR is None:
|
||||
_PROCESS_SCAN_ERROR = "proc-process-table-empty"
|
||||
if _PROCESS_SCAN_TRUNCATED or _OBSERVE_SCAN_TRUNCATED or _PROCESS_SCAN_ERROR or _OBSERVE_SCAN_ERROR:
|
||||
return []
|
||||
fresh_observer_pids = set()
|
||||
stale_observer_pids = set()
|
||||
for record in records:
|
||||
pid = safe_int(record.get("pid"))
|
||||
if pid <= 0 or pid not in processes:
|
||||
continue
|
||||
if record.get("staleLiveSignal"):
|
||||
stale_observer_pids.add(pid)
|
||||
else:
|
||||
fresh_observer_pids.add(pid)
|
||||
protected_pids = set()
|
||||
for pid in fresh_observer_pids:
|
||||
protected_pids.update(process_descendants(processes, pid))
|
||||
candidates = []
|
||||
for sid in sorted(set(safe_int(row.get("sid")) for row in processes.values() if safe_int(row.get("sid")) > 0)):
|
||||
closed = session_closed_tree(processes, sid)
|
||||
if closed is None:
|
||||
continue
|
||||
members = closed["members"]
|
||||
member_pids = set(members.keys())
|
||||
if member_pids.intersection(protected_pids):
|
||||
continue
|
||||
root = members[closed["rootPid"]]
|
||||
if safe_int(root.get("ageSeconds")) < stale_seconds:
|
||||
continue
|
||||
has_browser = any(command_matches(row, browser_patterns) for row in members.values())
|
||||
intersecting_stale = member_pids.intersection(stale_observer_pids)
|
||||
kind = None
|
||||
reason = None
|
||||
if intersecting_stale and command_matches(root, observer_patterns):
|
||||
kind = "stale-web-observer-process-tree"
|
||||
reason = "stale-observer-heartbeat"
|
||||
elif not member_pids.intersection(set(safe_int(item.get("pid")) for item in records if item.get("pidAlive"))):
|
||||
if command_matches(root, root_patterns) and has_browser:
|
||||
kind = "browser-orphan-process-tree"
|
||||
reason = "browser-session-without-observer-record"
|
||||
if kind is None:
|
||||
continue
|
||||
identities = [
|
||||
{
|
||||
"pid": pid,
|
||||
"ppid": safe_int(row.get("ppid")),
|
||||
"sid": safe_int(row.get("sid")),
|
||||
"startTicks": safe_int(row.get("startTicks")),
|
||||
"rssBytes": safe_int(row.get("rssBytes")),
|
||||
"commandPreview": row.get("commandPreview"),
|
||||
}
|
||||
for pid, row in sorted(members.items())
|
||||
]
|
||||
expected_memory = sum(safe_int(item.get("rssBytes")) for item in identities)
|
||||
candidates.append({
|
||||
"id": memory_process_candidate_id(kind, root),
|
||||
"kind": kind,
|
||||
"risk": "medium",
|
||||
"description": "Terminate a YAML-allowlisted stale WebProbe process session after identity and active-observer revalidation",
|
||||
"reason": reason,
|
||||
"rootPid": safe_int(root.get("pid")),
|
||||
"sid": sid,
|
||||
"rootStartTicks": safe_int(root.get("startTicks")),
|
||||
"processCount": len(identities),
|
||||
"processIdentities": identities,
|
||||
"expectedMemoryRssBytes": expected_memory,
|
||||
"expectedMemoryRss": fmt_bytes(expected_memory),
|
||||
"estimatedDiskBytes": 0,
|
||||
"estimatedReclaimBytes": 0,
|
||||
"activeObserverIntersection": False,
|
||||
"treeClosure": "unique-root-ppid-sid-closed",
|
||||
"action": {"op": "term-wait-kill-exact-process-tree", "allowlist": "yaml-web-probe-memory-pressure"},
|
||||
})
|
||||
max_candidates = config_int(MEMORY_CONFIG, "maxMemoryReclaimCandidates", 20, minimum=1, maximum=100)
|
||||
return sorted(candidates, key=lambda item: safe_int(item.get("expectedMemoryRssBytes")), reverse=True)[:max_candidates]
|
||||
|
||||
def memory_reclaim_scan_diagnostics():
|
||||
blocked = bool(_MEMORY_RECLAIM_CONFIG_ERROR or _OBSERVE_SCAN_TRUNCATED or _PROCESS_SCAN_TRUNCATED or _OBSERVE_SCAN_ERROR or _PROCESS_SCAN_ERROR)
|
||||
return {
|
||||
"ok": not blocked,
|
||||
"blocked": blocked,
|
||||
"configError": _MEMORY_RECLAIM_CONFIG_ERROR,
|
||||
"observeScanTruncated": bool(_OBSERVE_SCAN_TRUNCATED),
|
||||
"processScanTruncated": bool(_PROCESS_SCAN_TRUNCATED),
|
||||
"observeScanError": _OBSERVE_SCAN_ERROR,
|
||||
"processScanError": _PROCESS_SCAN_ERROR,
|
||||
"observeRoots": {
|
||||
"configuredCount": len(_OBSERVE_ROOT_DIAGNOSTICS.get("configured") or []),
|
||||
"readableCount": len(_OBSERVE_ROOT_DIAGNOSTICS.get("readable") or []),
|
||||
"missingCount": len(_OBSERVE_ROOT_DIAGNOSTICS.get("missing") or []),
|
||||
"minimumReadable": safe_int(MEMORY_CONFIG.get("minimumReadableObserveRoots")),
|
||||
"configured": _OBSERVE_ROOT_DIAGNOSTICS.get("configured") or [],
|
||||
"readable": _OBSERVE_ROOT_DIAGNOSTICS.get("readable") or [],
|
||||
"missing": _OBSERVE_ROOT_DIAGNOSTICS.get("missing") or [],
|
||||
},
|
||||
"policySource": "config/unidesk-cli.yaml#gc.remote.targets.%s.memoryPressure" % PROVIDER_ID,
|
||||
}
|
||||
|
||||
def matching_live_identity(identity, processes):
|
||||
pid = safe_int(identity.get("pid"))
|
||||
row = processes.get(pid)
|
||||
return row if row \
|
||||
and safe_int(row.get("startTicks")) == safe_int(identity.get("startTicks")) \
|
||||
and safe_int(row.get("ppid")) == safe_int(identity.get("ppid")) \
|
||||
and safe_int(row.get("sid")) == safe_int(identity.get("sid")) else None
|
||||
|
||||
def matching_surviving_identity(identity, processes):
|
||||
pid = safe_int(identity.get("pid"))
|
||||
row = processes.get(pid)
|
||||
return row if row \
|
||||
and safe_int(row.get("startTicks")) == safe_int(identity.get("startTicks")) else None
|
||||
|
||||
def require_complete_process_scan(processes, phase):
|
||||
if not processes or _PROCESS_SCAN_ERROR or _PROCESS_SCAN_TRUNCATED:
|
||||
raise RuntimeError("memory process identity scan blocked during %s: %s" % (
|
||||
phase,
|
||||
_PROCESS_SCAN_ERROR or "process-scan-truncated",
|
||||
))
|
||||
return processes
|
||||
|
||||
def extend_with_allowlisted_session_members(identities, processes, sid, browser_patterns):
|
||||
known = {(safe_int(item.get("pid")), safe_int(item.get("startTicks"))) for item in identities}
|
||||
added = []
|
||||
for pid, row in processes.items():
|
||||
identity_key = (safe_int(pid), safe_int(row.get("startTicks")))
|
||||
if identity_key in known or safe_int(row.get("sid")) != safe_int(sid) or not command_matches(row, browser_patterns):
|
||||
continue
|
||||
item = {
|
||||
"pid": safe_int(pid),
|
||||
"ppid": safe_int(row.get("ppid")),
|
||||
"sid": safe_int(row.get("sid")),
|
||||
"startTicks": safe_int(row.get("startTicks")),
|
||||
"rssBytes": safe_int(row.get("rssBytes")),
|
||||
"commandPreview": row.get("commandPreview"),
|
||||
"discoveredAfterTerm": True,
|
||||
}
|
||||
identities.append(item)
|
||||
added.append(item)
|
||||
known.add(identity_key)
|
||||
return added
|
||||
|
||||
def execute_memory_process_candidate(candidate):
|
||||
current = next((item for item in collect_memory_reclaim_candidates() if item.get("id") == candidate.get("id")), None)
|
||||
if current is None:
|
||||
raise RuntimeError("memory candidate no longer has the same closed identity or is now protected")
|
||||
before = collect_memory_snapshot()
|
||||
identities = [dict(item) for item in (current.get("processIdentities") or [])]
|
||||
candidate_sid = safe_int(current.get("sid"))
|
||||
browser_patterns = config_list(MEMORY_CONFIG, "browserProcessPatterns", [])
|
||||
late_members = []
|
||||
processes = require_complete_process_scan(read_proc_process_table(), "pre-term-revalidation")
|
||||
live = [item for item in identities if matching_live_identity(item, processes)]
|
||||
for item in sorted(live, key=lambda value: safe_int(value.get("pid")), reverse=True):
|
||||
try:
|
||||
os.kill(safe_int(item.get("pid")), signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
grace_seconds = config_int(MEMORY_CONFIG, "terminationGraceSeconds", 5, minimum=1, maximum=30)
|
||||
poll_ms = config_int(MEMORY_CONFIG, "terminationPollMilliseconds", 200, minimum=50, maximum=1000)
|
||||
deadline = time.time() + grace_seconds
|
||||
residual = []
|
||||
while time.time() < deadline:
|
||||
processes = require_complete_process_scan(read_proc_process_table(), "term-grace-poll")
|
||||
late_members.extend(extend_with_allowlisted_session_members(identities, processes, candidate_sid, browser_patterns))
|
||||
residual = [item for item in identities if matching_surviving_identity(item, processes)]
|
||||
if not residual:
|
||||
break
|
||||
time.sleep(float(poll_ms) / 1000.0)
|
||||
processes = require_complete_process_scan(read_proc_process_table(), "pre-kill-revalidation")
|
||||
late_members.extend(extend_with_allowlisted_session_members(identities, processes, candidate_sid, browser_patterns))
|
||||
residual = [item for item in identities if matching_surviving_identity(item, processes)]
|
||||
for item in sorted(residual, key=lambda value: safe_int(value.get("pid")), reverse=True):
|
||||
try:
|
||||
os.kill(safe_int(item.get("pid")), signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
final_deadline = time.time() + min(2.0, float(grace_seconds))
|
||||
remaining = residual
|
||||
while time.time() < final_deadline:
|
||||
processes = require_complete_process_scan(read_proc_process_table(), "kill-grace-poll")
|
||||
late_members.extend(extend_with_allowlisted_session_members(identities, processes, candidate_sid, browser_patterns))
|
||||
remaining = [item for item in identities if matching_surviving_identity(item, processes)]
|
||||
if not remaining:
|
||||
break
|
||||
time.sleep(float(poll_ms) / 1000.0)
|
||||
after = collect_memory_snapshot()
|
||||
before_available = safe_int((before.get("memory") or {}).get("availableBytes"), None)
|
||||
after_available = safe_int((after.get("memory") or {}).get("availableBytes"), None)
|
||||
return {
|
||||
"status": "failed" if remaining else "succeeded",
|
||||
"reclaimedBytes": 0,
|
||||
"reclaimedMemoryBytes": max(0, after_available - before_available) if before_available is not None and after_available is not None else None,
|
||||
"expectedMemoryRssBytes": current.get("expectedMemoryRssBytes"),
|
||||
"termSignalCount": len(live),
|
||||
"killSignalCount": len(residual),
|
||||
"remainingIdentityCount": len(remaining),
|
||||
"lateAllowlistedIdentityCount": len(late_members),
|
||||
"remainingIdentities": [
|
||||
{
|
||||
"pid": safe_int(item.get("pid")),
|
||||
"startTicks": safe_int(item.get("startTicks")),
|
||||
"sid": safe_int(item.get("sid")),
|
||||
"originalPpid": safe_int(item.get("ppid")),
|
||||
"observedPpid": safe_int((matching_surviving_identity(item, processes) or {}).get("ppid")),
|
||||
}
|
||||
for item in remaining
|
||||
],
|
||||
"memoryBefore": before,
|
||||
"memoryAfter": after,
|
||||
"identityRevalidated": True,
|
||||
"activeObserverIntersection": False,
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ interface RemoteGcOptions {
|
||||
full: boolean;
|
||||
historyLimit: number;
|
||||
saveSnapshot: boolean;
|
||||
memoryPressureOnly: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
|
||||
@@ -65,6 +66,7 @@ const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
|
||||
full: false,
|
||||
historyLimit: 12,
|
||||
saveSnapshot: true,
|
||||
memoryPressureOnly: false,
|
||||
};
|
||||
|
||||
const GC_CONFIG_RELATIVE_PATH = "config/unidesk-cli.yaml";
|
||||
@@ -124,6 +126,16 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
};
|
||||
}
|
||||
const options = parseRemoteGcOptions(args);
|
||||
if (options.memoryPressureOnly && subaction !== "plan" && subaction !== "run") {
|
||||
return {
|
||||
ok: false,
|
||||
error: "gc-remote-memory-pressure-only-action-invalid",
|
||||
action: subaction,
|
||||
supportedActions: ["plan", "run"],
|
||||
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan --memory-pressure-only`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm --memory-pressure-only`,
|
||||
};
|
||||
}
|
||||
if (subaction === "memory") return await runRemoteGc(config, providerId, "memory", options);
|
||||
if (subaction === "plan" || subaction === "dry-run") return await runRemoteGc(config, providerId, "plan", options);
|
||||
if (subaction === "snapshot" || subaction === "growth") return await runRemoteGc(config, providerId, "snapshot", options);
|
||||
@@ -137,8 +149,8 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
dryRun: true,
|
||||
mutation: false,
|
||||
requiredFlag: "--confirm",
|
||||
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm`,
|
||||
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`,
|
||||
};
|
||||
}
|
||||
return await runRemoteGc(config, providerId, "run", options);
|
||||
@@ -239,6 +251,8 @@ function parseRemoteGcOptions(args: string[]): RemoteGcOptions {
|
||||
options.full = true;
|
||||
} else if (arg === "--no-save" || arg === "--no-snapshot-save") {
|
||||
options.saveSnapshot = false;
|
||||
} else if (arg === "--memory-pressure-only") {
|
||||
options.memoryPressureOnly = true;
|
||||
} else {
|
||||
throw new Error(`unknown gc remote option: ${arg}`);
|
||||
}
|
||||
|
||||
@@ -517,6 +517,10 @@ function gcHelp(): unknown {
|
||||
"bun scripts/cli.ts gc policy plan",
|
||||
"bun scripts/cli.ts gc policy install",
|
||||
"bun scripts/cli.ts gc remote G14 plan",
|
||||
"bun scripts/cli.ts gc remote NC01 memory",
|
||||
"bun scripts/cli.ts gc remote NC01 plan --memory-pressure-only",
|
||||
"bun scripts/cli.ts gc remote NC01 run --confirm --memory-pressure-only",
|
||||
"bun scripts/cli.ts gc remote NC01 status --job-id <job-id>",
|
||||
"bun scripts/cli.ts gc remote G14 plan --target-use-percent 50 --include-hwlab-registry",
|
||||
"bun scripts/cli.ts gc remote G14 run --confirm",
|
||||
"bun scripts/cli.ts gc remote G14 status --job-id <id>",
|
||||
@@ -563,6 +567,7 @@ function gcHelp(): unknown {
|
||||
"db-trace run --vacuum-full": "rewrite public.oa_events after deletion so df can reclaim disk; requires maintenance window",
|
||||
"policy plan|install": "render or install journald caps and a daily low-risk gc systemd timer from config/unidesk-cli.yaml#gc.policyTimer",
|
||||
"remote <providerId> plan|run": "run bounded GC through UniDesk SSH passthrough on a provider host; G14 protects HWLAB k3s/runtime/PVC/workspace paths, and HWLAB registry retention is explicit opt-in with workload-ref, digest-closure, recent-tag and per-repo tag protection",
|
||||
"remote <providerId> memory": "只读输出目标节点 /proc/meminfo 的 MemAvailable；不合并 swap、cgroup 或 virtual memory，也不执行 GC mutation",
|
||||
"--no-file-logs|--no-docker-logs|--no-journal|--no-build-cache|--no-tmp|--no-db-summary": "disable one collector",
|
||||
},
|
||||
reference: "docs/reference/gc.md",
|
||||
|
||||
@@ -167,6 +167,10 @@ export interface HwlabRuntimeWebProbeMemoryStartGuardSpec {
|
||||
readonly manualComparator: "less-than-or-equal";
|
||||
readonly sentinelComparator: "less-than";
|
||||
readonly snapshotTimeoutSeconds: number;
|
||||
readonly launchLockPath: string;
|
||||
readonly launchLockTimeoutSeconds: number;
|
||||
readonly launchReadyTimeoutSeconds: number;
|
||||
readonly launchReadyPollMilliseconds: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeObserverLifecycleSpec {
|
||||
@@ -176,8 +180,24 @@ export interface HwlabRuntimeWebProbeObserverLifecycleSpec {
|
||||
readonly maxPerformanceEntries: number;
|
||||
readonly maxBrowserProcessHistoryEntries: number;
|
||||
readonly maxBrowserFreezeSignalHistoryEntries: number;
|
||||
readonly maxRealtimeEventEntries: number;
|
||||
readonly maxRealtimeEventBodyBytes: number;
|
||||
readonly maxDomEvidenceRows: number;
|
||||
readonly maxProcessScanEntries: number;
|
||||
readonly maxPerformanceCaptureSeconds: number;
|
||||
readonly maxResponseBodyBytes: number;
|
||||
readonly maxWebPerformanceBodyBytes: number;
|
||||
readonly maxJsonlQueueEntries: number;
|
||||
readonly maxJsonlQueueBytes: number;
|
||||
readonly maxJsonlLineBytes: number;
|
||||
readonly jsonlFlushTimeoutMs: number;
|
||||
readonly maxResponseBodyInFlight: number;
|
||||
readonly maxPassiveListenerTasks: number;
|
||||
readonly passiveTaskFlushTimeoutMs: number;
|
||||
readonly closeStepTimeoutMs: number;
|
||||
readonly maxProcessReadConcurrency: number;
|
||||
readonly maxRealtimeTotalEntries: number;
|
||||
readonly maxRealtimeTotalBodyBytes: number;
|
||||
}
|
||||
|
||||
export interface HwlabRuntimeWebProbeResourcePolicySpec {
|
||||
@@ -1471,6 +1491,10 @@ function webProbeResourcePolicyConfig(value: unknown, path: string): HwlabRuntim
|
||||
manualComparator: enumStringField(memory, "manualComparator", `${path}.memoryStartGuard`, ["less-than-or-equal"] as const),
|
||||
sentinelComparator: enumStringField(memory, "sentinelComparator", `${path}.memoryStartGuard`, ["less-than"] as const),
|
||||
snapshotTimeoutSeconds: boundedIntegerField(memory, "snapshotTimeoutSeconds", `${path}.memoryStartGuard`, 5, 120),
|
||||
launchLockPath: absoluteHostPathField(stringField(memory, "launchLockPath", `${path}.memoryStartGuard`), `${path}.memoryStartGuard.launchLockPath`),
|
||||
launchLockTimeoutSeconds: boundedIntegerField(memory, "launchLockTimeoutSeconds", `${path}.memoryStartGuard`, 1, 60),
|
||||
launchReadyTimeoutSeconds: boundedIntegerField(memory, "launchReadyTimeoutSeconds", `${path}.memoryStartGuard`, 1, 30),
|
||||
launchReadyPollMilliseconds: boundedIntegerField(memory, "launchReadyPollMilliseconds", `${path}.memoryStartGuard`, 50, 1000),
|
||||
},
|
||||
observerLifecycle: {
|
||||
maxRunSeconds: boundedIntegerField(lifecycle, "maxRunSeconds", `${path}.observerLifecycle`, 1, 86_400),
|
||||
@@ -1479,8 +1503,24 @@ function webProbeResourcePolicyConfig(value: unknown, path: string): HwlabRuntim
|
||||
maxPerformanceEntries: boundedIntegerField(lifecycle, "maxPerformanceEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxBrowserProcessHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserProcessHistoryEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxBrowserFreezeSignalHistoryEntries: boundedIntegerField(lifecycle, "maxBrowserFreezeSignalHistoryEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxRealtimeEventEntries: boundedIntegerField(lifecycle, "maxRealtimeEventEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxRealtimeEventBodyBytes: boundedIntegerField(lifecycle, "maxRealtimeEventBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
|
||||
maxDomEvidenceRows: boundedIntegerField(lifecycle, "maxDomEvidenceRows", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxProcessScanEntries: boundedIntegerField(lifecycle, "maxProcessScanEntries", `${path}.observerLifecycle`, 1, 1_000_000),
|
||||
maxPerformanceCaptureSeconds: boundedIntegerField(lifecycle, "maxPerformanceCaptureSeconds", `${path}.observerLifecycle`, 1, 600),
|
||||
maxResponseBodyBytes: boundedIntegerField(lifecycle, "maxResponseBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
|
||||
maxWebPerformanceBodyBytes: boundedIntegerField(lifecycle, "maxWebPerformanceBodyBytes", `${path}.observerLifecycle`, 1, 16 * 1024 * 1024),
|
||||
maxJsonlQueueEntries: boundedIntegerField(lifecycle, "maxJsonlQueueEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxJsonlQueueBytes: boundedIntegerField(lifecycle, "maxJsonlQueueBytes", `${path}.observerLifecycle`, 1024, 64 * 1024 * 1024),
|
||||
maxJsonlLineBytes: boundedIntegerField(lifecycle, "maxJsonlLineBytes", `${path}.observerLifecycle`, 1024, 16 * 1024 * 1024),
|
||||
jsonlFlushTimeoutMs: boundedIntegerField(lifecycle, "jsonlFlushTimeoutMs", `${path}.observerLifecycle`, 100, 60_000),
|
||||
maxResponseBodyInFlight: boundedIntegerField(lifecycle, "maxResponseBodyInFlight", `${path}.observerLifecycle`, 1, 64),
|
||||
maxPassiveListenerTasks: boundedIntegerField(lifecycle, "maxPassiveListenerTasks", `${path}.observerLifecycle`, 1, 10_000),
|
||||
passiveTaskFlushTimeoutMs: boundedIntegerField(lifecycle, "passiveTaskFlushTimeoutMs", `${path}.observerLifecycle`, 100, 60_000),
|
||||
closeStepTimeoutMs: boundedIntegerField(lifecycle, "closeStepTimeoutMs", `${path}.observerLifecycle`, 100, 60_000),
|
||||
maxProcessReadConcurrency: boundedIntegerField(lifecycle, "maxProcessReadConcurrency", `${path}.observerLifecycle`, 1, 256),
|
||||
maxRealtimeTotalEntries: boundedIntegerField(lifecycle, "maxRealtimeTotalEntries", `${path}.observerLifecycle`, 1, 100_000),
|
||||
maxRealtimeTotalBodyBytes: boundedIntegerField(lifecycle, "maxRealtimeTotalBodyBytes", `${path}.observerLifecycle`, 1024, 64 * 1024 * 1024),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -379,7 +379,7 @@ async function authenticate(browserContext) {
|
||||
await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item?.retryLabel || retryLabel, retryAttempt: attempt, retryMaxAttempts: maxAttempts, retryDelayMs: item?.retryDelayMs ?? 0, lastStatus: item?.status ?? 0, lastStatusText: item?.statusText ?? "request-error", retryable, cookiePresent: false, retryExhausted: false, lastError: item?.error || null, valuesRedacted: true } }).catch(() => {});
|
||||
if (!retryable) break;
|
||||
}
|
||||
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs);
|
||||
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await interruptibleRunnerSleep(retryDelayMs);
|
||||
}
|
||||
const cookieState = await readAuthCookieState(browserContext);
|
||||
const last = attempts[attempts.length - 1] || null;
|
||||
@@ -414,12 +414,13 @@ async function pageAuthLogin(browserContext, loginUrl, credential = { username,
|
||||
headers: { accept: "application/json", "content-type": "application/json" },
|
||||
timeout: authLoginRequestTimeoutMs,
|
||||
});
|
||||
await response.text().catch(() => "");
|
||||
return {
|
||||
const result = {
|
||||
ok: response.ok(),
|
||||
status: response.status(),
|
||||
statusText: response.statusText() || "",
|
||||
};
|
||||
if (typeof response.dispose === "function") await response.dispose().catch(() => {});
|
||||
return result;
|
||||
}
|
||||
|
||||
async function loginAccount(command) {
|
||||
@@ -479,7 +480,7 @@ async function loginAccount(command) {
|
||||
cookieState = await readAuthCookieState(context).catch(() => ({ cookiePresent: false, cookieNames: [] }));
|
||||
if (!retryable) break;
|
||||
}
|
||||
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs);
|
||||
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await interruptibleRunnerSleep(retryDelayMs);
|
||||
}
|
||||
response = response ?? { ok: false, status: 0, statusText: "api-login-failed" };
|
||||
cookieState = cookieState ?? await readAuthCookieState(context);
|
||||
@@ -514,7 +515,6 @@ async function logoutAccount(command = {}) {
|
||||
const logoutUrl = new URL("/logout", baseUrl).toString();
|
||||
const response = await page.evaluate(async (input) => {
|
||||
const res = await fetch(input.logoutUrl, { method: "POST", headers: { accept: "application/json" }, credentials: "include" });
|
||||
await res.text().catch(() => "");
|
||||
return { ok: res.ok, status: res.status, statusText: res.statusText || "" };
|
||||
}, { logoutUrl });
|
||||
await context.clearCookies().catch(() => {});
|
||||
@@ -1295,11 +1295,9 @@ async function createSessionFromUi() {
|
||||
const createStatus = createResponse.status();
|
||||
let createPayload = null;
|
||||
let createPayloadError = null;
|
||||
try {
|
||||
createPayload = await createResponse.json();
|
||||
} catch (error) {
|
||||
createPayloadError = errorSummary(error);
|
||||
}
|
||||
const createBody = await boundedResponseJson(createResponse);
|
||||
createPayload = createBody.value;
|
||||
createPayloadError = createBody.error;
|
||||
const createdSessionId = sessionIdFromAgentSessionPayload(createPayload);
|
||||
if (createStatus < 200 || createStatus >= 300 || !createdSessionId) {
|
||||
const error = new Error("newSession did not receive an authoritative session id from POST /v1/agent/sessions");
|
||||
@@ -1753,11 +1751,9 @@ async function sendPrompt(text, options = {}) {
|
||||
}
|
||||
let chatPayload = null;
|
||||
let chatPayloadError = null;
|
||||
try {
|
||||
chatPayload = await chatResponse.json();
|
||||
} catch (error) {
|
||||
chatPayloadError = errorSummary(error);
|
||||
}
|
||||
const chatBody = await boundedResponseJson(chatResponse);
|
||||
chatPayload = chatBody.value;
|
||||
chatPayloadError = chatBody.error;
|
||||
const payloadText = chatPayload ? JSON.stringify(chatPayload) : "";
|
||||
const traceId = payloadText.match(/\btrc_[A-Za-z0-9_-]+\b/u)?.[0] || null;
|
||||
const otelTraceId = typeof chatPayload?.otelTrace?.traceId === "string" && /^[0-9a-f]{32}$/u.test(chatPayload.otelTrace.traceId)
|
||||
|
||||
@@ -148,6 +148,7 @@ async function validateWorkbenchKafkaDebugReplay(command) {
|
||||
try { url = new URL(request.url()); } catch { return; }
|
||||
if (url.pathname === config.productEventsPath) {
|
||||
productEventSourceRequests.push({ method: request.method(), path: url.pathname, phase: requestPhase, valuesRedacted: true });
|
||||
if (productEventSourceRequests.length > maxRealtimeEventEntries) productEventSourceRequests.splice(0, productEventSourceRequests.length - maxRealtimeEventEntries);
|
||||
return;
|
||||
}
|
||||
if (url.pathname !== config.debugEventsPath) return;
|
||||
@@ -160,6 +161,7 @@ async function validateWorkbenchKafkaDebugReplay(command) {
|
||||
phase: requestPhase,
|
||||
valuesRedacted: true
|
||||
});
|
||||
if (requests.length > maxRealtimeEventEntries) requests.splice(0, requests.length - maxRealtimeEventEntries);
|
||||
};
|
||||
try {
|
||||
const controlRecovery = await ensureControlPageResponsiveForCommand("validateWorkbenchKafkaDebugReplay");
|
||||
@@ -386,7 +388,7 @@ async function workbenchKafkaDebugReplayWaitForCleared(panel, countsNode, timeou
|
||||
if (status === "idle" && counts?.appliedCount === 0 && counts?.receivedCount === 0) {
|
||||
return { status, ...counts, valuesRedacted: true };
|
||||
}
|
||||
await sleep(100);
|
||||
await interruptibleRunnerSleep(100);
|
||||
}
|
||||
throw new Error("Workbench Kafka debug panel did not clear to idle 0/0");
|
||||
}
|
||||
@@ -397,7 +399,7 @@ async function workbenchKafkaDebugReplayWaitForTerminal(panel, timeoutMs) {
|
||||
const status = await panel.getAttribute("data-replay-status");
|
||||
if (status === "completed") return status;
|
||||
if (status === "incomplete" || status === "error" || status === "closed") return status;
|
||||
await sleep(200);
|
||||
await interruptibleRunnerSleep(200);
|
||||
}
|
||||
throw new Error("Workbench Kafka debug replay did not reach a terminal panel status within " + timeoutMs + "ms");
|
||||
}
|
||||
@@ -406,7 +408,7 @@ async function workbenchKafkaDebugReplayWaitForTabSelected(tab, timeoutMs) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() <= deadline) {
|
||||
if (await tab.getAttribute("aria-selected") === "true") return true;
|
||||
await sleep(50);
|
||||
await interruptibleRunnerSleep(50);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -514,7 +516,7 @@ async function workbenchKafkaDebugReplayWaitForStablePanel(panel, countsNode, ti
|
||||
stableSinceMs = null;
|
||||
}
|
||||
previousSignature = signature;
|
||||
await sleep(100);
|
||||
await interruptibleRunnerSleep(100);
|
||||
}
|
||||
return {
|
||||
...latest,
|
||||
@@ -586,14 +588,14 @@ async function workbenchKafkaDebugReplayInspectRawWindow(input) {
|
||||
await rawCountsNode.waitFor({ state: "visible", timeout: 5000 });
|
||||
await rawTextNode.waitFor({ state: "visible", timeout: 5000 });
|
||||
await rawContractNode.waitFor({ state: "visible", timeout: 5000 });
|
||||
await sleep(input.config.requestObservationQuietMs);
|
||||
await interruptibleRunnerSleep(input.config.requestObservationQuietMs);
|
||||
const counts = workbenchKafkaDebugReplayRawCounts(await rawCountsNode.textContent());
|
||||
const rawText = await rawTextNode.inputValue();
|
||||
const contractText = String(await rawContractNode.textContent() || "").replace(/\s+/gu, " ").trim();
|
||||
input.setRequestPhase("trace-tab-restore");
|
||||
await traceTab.click({ timeout: 10000 });
|
||||
if (!await workbenchKafkaDebugReplayWaitForTabSelected(traceTab, 5000)) throw new Error("Workbench Trace debug tab did not restore after raw inspection");
|
||||
await sleep(input.config.requestObservationQuietMs);
|
||||
await interruptibleRunnerSleep(input.config.requestObservationQuietMs);
|
||||
input.setRequestPhase("complete");
|
||||
const pageUrlAfter = currentPageUrl();
|
||||
const productRequests = input.productEventSourceRequests.slice(requestStart);
|
||||
|
||||
@@ -227,7 +227,7 @@ function compactPerformanceAttribution(item) {
|
||||
}
|
||||
|
||||
async function capturePerformanceProfile(command) {
|
||||
const durationMs = boundedInteger(command.durationMs, 5000, 100, 600000);
|
||||
const durationMs = boundedInteger(command.durationMs, Math.min(5000, maxPerformanceCaptureMs), 100, maxPerformanceCaptureMs);
|
||||
const timeoutMs = Math.max(3000, Math.min(15000, Number(alertThresholds.playwrightResponsivenessRedMs) || 5000));
|
||||
const captureId = safeId(command.label || command.id || ("perf-" + Date.now().toString(36))) || ("perf-" + Date.now().toString(36));
|
||||
const targetPage = page && !page.isClosed() ? page : observerPage && !observerPage.isClosed() ? observerPage : null;
|
||||
@@ -264,7 +264,7 @@ async function capturePerformanceProfile(command) {
|
||||
beforeDrain,
|
||||
valuesRedacted: true,
|
||||
}));
|
||||
await sleep(durationMs);
|
||||
await interruptibleRunnerSleep(durationMs);
|
||||
stopped = await withHardTimeout(session.send("Profiler.stop"), Math.max(timeoutMs, 5000), "Profiler.stop exceeded " + Math.max(timeoutMs, 5000) + "ms");
|
||||
} catch (error) {
|
||||
captureError = error;
|
||||
|
||||
@@ -8,14 +8,26 @@ test("realtime fanout keeps the YAML terminal timeout when durationMs is absent"
|
||||
const build = new Function(
|
||||
"boundedInteger",
|
||||
"sanitize",
|
||||
"maxRealtimeTotalEntries",
|
||||
"maxRealtimeTotalBodyBytes",
|
||||
"maxRealtimeEventEntries",
|
||||
"maxRealtimeEventBodyBytes",
|
||||
`${source}\nreturn realtimeFanoutEffectiveProfile;`,
|
||||
) as (
|
||||
boundedInteger: (value: unknown, fallback: number, min: number, max: number) => number,
|
||||
sanitize: <T>(value: T) => T,
|
||||
maxRealtimeTotalEntries: number,
|
||||
maxRealtimeTotalBodyBytes: number,
|
||||
maxRealtimeEventEntries: number,
|
||||
maxRealtimeEventBodyBytes: number,
|
||||
) => (profile: Record<string, unknown>, durationMs: unknown) => Record<string, unknown>;
|
||||
const effectiveProfile = build(
|
||||
(value, fallback, min, max) => Math.min(max, Math.max(min, Number.isFinite(Number(value)) ? Number(value) : fallback)),
|
||||
(value) => value,
|
||||
512,
|
||||
4 * 1024 * 1024,
|
||||
256,
|
||||
64 * 1024,
|
||||
);
|
||||
const profile = {
|
||||
subscriberCount: 2,
|
||||
@@ -48,6 +60,9 @@ test("realtime fanout keeps the YAML terminal timeout when durationMs is absent"
|
||||
assert.equal(effectiveProfile(profile, undefined).terminalTimeoutMs, 480_000);
|
||||
assert.equal(effectiveProfile(profile, 5_000).terminalTimeoutMs, 5_000);
|
||||
assert.deepEqual(effectiveProfile(profile, null).conditionalEventTypePairs, profile.conditionalEventTypePairs);
|
||||
const budget = effectiveProfile(profile, null).aggregateBudget as Record<string, number>;
|
||||
assert.ok(budget.perArrayEntries * budget.entrySlots <= budget.maxTotalEntries);
|
||||
assert.ok(budget.perEventBodyBytes * budget.bodyRetentionSlots <= budget.maxTotalBodyBytes);
|
||||
});
|
||||
|
||||
test("realtime fanout terminal completion does not require absent optional tool families", () => {
|
||||
@@ -137,12 +152,12 @@ test("realtime fanout validates optional tool events only when they occur", () =
|
||||
test("realtime fanout retries a YAML-bounded transient health navigation", async () => {
|
||||
const source = nodeWebObserveRunnerRealtimeSource();
|
||||
const build = new Function(
|
||||
"sleep",
|
||||
"interruptibleRunnerSleep",
|
||||
"isRetryableNavigationError",
|
||||
"baseUrl",
|
||||
`${source}\nreturn realtimeGotoHealth;`,
|
||||
) as (
|
||||
sleep: (milliseconds: number) => Promise<void>,
|
||||
interruptibleRunnerSleep: (milliseconds: number) => Promise<void>,
|
||||
isRetryableNavigationError: (message: string) => boolean,
|
||||
baseUrl: string,
|
||||
) => (page: { goto: () => Promise<{ status: () => number }> }, timeoutMs: number, profile: Record<string, number>, label: string) => Promise<Record<string, unknown>>;
|
||||
@@ -433,11 +448,11 @@ test("realtime replay joint quiet waits for a late formal Kafka envelope before
|
||||
let kafkaRows = [first];
|
||||
const productRows = [first, late];
|
||||
const build = new Function(
|
||||
"sleep",
|
||||
"interruptibleRunnerSleep",
|
||||
"Date",
|
||||
`${source}\nreturn realtimeWaitReplayedTraceJointQuiet;`,
|
||||
) as (
|
||||
sleep: () => Promise<void>,
|
||||
interruptibleRunnerSleep: () => Promise<void>,
|
||||
date: { now: () => number },
|
||||
) => (
|
||||
debug: Record<string, any>,
|
||||
@@ -495,11 +510,11 @@ test("realtime replay joint quiet rejects a permanent 31/32 split with bounded i
|
||||
const first = replayEnvelope("evt_first", 1);
|
||||
const late = replayEnvelope("evt_late", 2, true);
|
||||
const build = new Function(
|
||||
"sleep",
|
||||
"interruptibleRunnerSleep",
|
||||
"Date",
|
||||
`${source}\nreturn realtimeWaitReplayedTraceJointQuiet;`,
|
||||
) as (
|
||||
sleep: () => Promise<void>,
|
||||
interruptibleRunnerSleep: () => Promise<void>,
|
||||
date: { now: () => number },
|
||||
) => (
|
||||
debug: Record<string, any>,
|
||||
|
||||
@@ -23,6 +23,8 @@ async function validateRealtimeFanout(command) {
|
||||
if (!provider) throw new Error("validateRealtimeFanout requires provider");
|
||||
if (!firstPrompt || !secondPrompt) throw new Error("validateRealtimeFanout requires first and second prompt");
|
||||
const effective = realtimeFanoutEffectiveProfile(profile, command.durationMs);
|
||||
const aggregateCounters = { networkDroppedEntries: 0, valuesRedacted: true };
|
||||
realtimeAggregateLast = { commandId: command.id, profile: profileId, budget: effective.aggregateBudget, counters: aggregateCounters, status: "running", valuesRedacted: true };
|
||||
const reportDir = path.join(stateDir, "artifacts", "realtime-fanout", safeId(command.id));
|
||||
await mkdir(reportDir, { recursive: true, mode: 0o700 });
|
||||
const network = [];
|
||||
@@ -40,7 +42,10 @@ async function validateRealtimeFanout(command) {
|
||||
lastEventId: request.headers()["last-event-id"] || null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
if (network.length > 1000) network.splice(0, network.length - 1000);
|
||||
if (network.length > effective.aggregateBudget.perArrayEntries) {
|
||||
aggregateCounters.networkDroppedEntries += network.length - effective.aggregateBudget.perArrayEntries;
|
||||
network.splice(0, network.length - effective.aggregateBudget.perArrayEntries);
|
||||
}
|
||||
};
|
||||
page.on("request", requestListener);
|
||||
let subscriberA = null;
|
||||
@@ -130,7 +135,7 @@ async function validateRealtimeFanout(command) {
|
||||
subscriberB2 = await realtimeNewSubscriber(storageState, sessionId, "subscriber-b-reconnected", effective);
|
||||
const bReconnected = await realtimeWaitProductConnected(subscriberB2, effective.barrierTimeoutMs);
|
||||
realtimeAssertConnectedContract(bReconnected, sessionId, "subscriber-b-reconnected", effective, { requireRetainedEvents: effective.expectedProductSse.replayPriorTraceOnReconnect });
|
||||
await sleep(effective.reconnectQuietMs);
|
||||
await interruptibleRunnerSleep(effective.reconnectQuietMs);
|
||||
let reconnectBeforeSecond;
|
||||
if (effective.expectedProductSse.replayPriorTraceOnReconnect) {
|
||||
const firstReplayBeforeSecond = await realtimeWaitReplayedTraceJointQuiet(
|
||||
@@ -258,6 +263,13 @@ async function validateRealtimeFanout(command) {
|
||||
realtimeTurnSummary(first, firstDebug, productA, firstTerminal),
|
||||
realtimeTurnSummary(second, secondDebug, productB2, secondTerminalB),
|
||||
];
|
||||
aggregateCounters.productDroppedEntries = Number(productA.droppedEntries || 0) + Number(productB2.droppedEntries || 0);
|
||||
aggregateCounters.debugDroppedEntries = Number(firstDebug.droppedEntries || 0) + Number(secondDebug.droppedEntries || 0);
|
||||
aggregateCounters.productDroppedBodyEntries = Number(productA.droppedBodyEntries || 0) + Number(productB2.droppedBodyEntries || 0);
|
||||
aggregateCounters.productDroppedBodyBytes = Number(productA.droppedBodyBytes || 0) + Number(productB2.droppedBodyBytes || 0);
|
||||
aggregateCounters.debugDroppedBodyEntries = Number(firstDebug.droppedBodyEntries || 0) + Number(secondDebug.droppedBodyEntries || 0);
|
||||
aggregateCounters.debugDroppedBodyBytes = Number(firstDebug.droppedBodyBytes || 0) + Number(secondDebug.droppedBodyBytes || 0);
|
||||
realtimeAggregateLast = { ...realtimeAggregateLast, status: "completed", counters: aggregateCounters, valuesRedacted: true };
|
||||
Object.assign(report, {
|
||||
ok: true,
|
||||
status: "passed",
|
||||
@@ -290,6 +302,7 @@ async function validateRealtimeFanout(command) {
|
||||
valuesRedacted: true,
|
||||
},
|
||||
forbiddenRequests: { count: forbidden.length, rows: forbidden, valuesRedacted: true },
|
||||
aggregateBudget: { ...effective.aggregateBudget, counters: aggregateCounters, valuesRedacted: true },
|
||||
screenshots: [bBeforeShot, summaryScreenshot, subscriberAScreenshot, subscriberB2Screenshot],
|
||||
warmRunnerReused: first.runnerId && second.runnerId ? first.runnerId === second.runnerId : null,
|
||||
valuesRedacted: true,
|
||||
@@ -326,6 +339,7 @@ async function validateRealtimeFanout(command) {
|
||||
valuesRedacted: true,
|
||||
};
|
||||
} catch (error) {
|
||||
realtimeAggregateLast = { ...realtimeAggregateLast, status: "failed", counters: aggregateCounters, failure: errorSummary(error), valuesRedacted: true };
|
||||
Object.assign(report, {
|
||||
ok: false,
|
||||
status: "failed",
|
||||
@@ -367,6 +381,8 @@ async function validateExistingSessionRefresh(command) {
|
||||
const profile = realtimeFanoutProfiles[profileId];
|
||||
if (!profile || typeof profile !== "object") throw new Error("validateExistingSessionRefresh profile is not declared by the owning YAML: " + profileId);
|
||||
const effective = realtimeFanoutEffectiveProfile(profile, command.durationMs);
|
||||
const aggregateCounters = { networkDroppedEntries: 0, valuesRedacted: true };
|
||||
realtimeAggregateLast = { commandId: command.id, profile: profileId, budget: effective.aggregateBudget, counters: aggregateCounters, status: "running", valuesRedacted: true };
|
||||
if (effective.expectedProductSse.refreshHandoff !== true) throw new Error("validateExistingSessionRefresh requires a YAML profile with refresh handoff enabled");
|
||||
const reportDir = path.join(stateDir, "artifacts", "existing-session-refresh", safeId(command.id));
|
||||
await mkdir(reportDir, { recursive: true, mode: 0o700 });
|
||||
@@ -385,7 +401,10 @@ async function validateExistingSessionRefresh(command) {
|
||||
lastEventId: request.headers()["last-event-id"] || null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
if (network.length > 1000) network.splice(0, network.length - 1000);
|
||||
if (network.length > effective.aggregateBudget.perArrayEntries) {
|
||||
aggregateCounters.networkDroppedEntries += network.length - effective.aggregateBudget.perArrayEntries;
|
||||
network.splice(0, network.length - effective.aggregateBudget.perArrayEntries);
|
||||
}
|
||||
};
|
||||
const report = {
|
||||
ok: false,
|
||||
@@ -447,7 +466,7 @@ async function validateExistingSessionRefresh(command) {
|
||||
}
|
||||
phase = "live-handoff";
|
||||
await realtimeWaitForSameWorkbenchDom(before, effective.barrierTimeoutMs);
|
||||
await sleep(effective.reconnectQuietMs);
|
||||
await interruptibleRunnerSleep(effective.reconnectQuietMs);
|
||||
const after = await realtimeWorkbenchDomIdentity();
|
||||
realtimeAssertSameWorkbenchDom(before, after);
|
||||
const browserEvidence = await page.evaluate(() => JSON.parse(JSON.stringify(window.__unideskExistingSessionRefresh || {})));
|
||||
@@ -463,11 +482,16 @@ async function validateExistingSessionRefresh(command) {
|
||||
if (mainStreams[0].traceId !== null || mainStreams[0].afterSeq !== null || mainStreams[0].lastEventId !== null) throw new Error("main Workbench product SSE used a trace or cursor scope");
|
||||
if (forbidden.length > 0) throw new Error("forbidden recovery or polling requests observed: " + forbidden.map((item) => item.path).join(","));
|
||||
phase = "evidence-write";
|
||||
const screenshotFile = path.join(reportDir, "existing-session-refresh.png");
|
||||
await page.screenshot({ path: screenshotFile, fullPage: true, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
|
||||
const screenshotMeta = await fileMeta(screenshotFile);
|
||||
screenshot = { kind: "existing-session-refresh-screenshot", path: screenshotFile, ...screenshotMeta, commandId: activeCommandId, valuesRedacted: true };
|
||||
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...screenshot });
|
||||
const screenshotSkipped = reserveScreenshot("existing-session-refresh");
|
||||
if (screenshotSkipped !== null) {
|
||||
screenshot = screenshotSkipped;
|
||||
} else {
|
||||
const screenshotFile = path.join(reportDir, "existing-session-refresh.png");
|
||||
await page.screenshot({ path: screenshotFile, fullPage: false, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
|
||||
const screenshotMeta = await fileMeta(screenshotFile);
|
||||
screenshot = { kind: "existing-session-refresh-screenshot", path: screenshotFile, ...screenshotMeta, commandId: activeCommandId, valuesRedacted: true };
|
||||
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...screenshot });
|
||||
}
|
||||
const productSse = {
|
||||
requestCount: mainStreams.length,
|
||||
browserSourceCount: browserEvidence.sources.length,
|
||||
@@ -488,10 +512,12 @@ async function validateExistingSessionRefresh(command) {
|
||||
connected: connectedSummary,
|
||||
refreshReplay,
|
||||
productSse,
|
||||
aggregateBudget: { ...effective.aggregateBudget, counters: { ...aggregateCounters, browserDroppedEntries: Number(browserEvidence.droppedEntries || 0), browserDroppedBodyEntries: Number(browserEvidence.droppedBodyEntries || 0), browserDroppedBodyBytes: Number(browserEvidence.droppedBodyBytes || 0) }, valuesRedacted: true },
|
||||
forbiddenRequests: { count: 0, rows: [], valuesRedacted: true },
|
||||
screenshot,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
realtimeAggregateLast = { ...realtimeAggregateLast, status: "completed", counters: { ...aggregateCounters, browserDroppedEntries: Number(browserEvidence.droppedEntries || 0), browserDroppedBodyEntries: Number(browserEvidence.droppedBodyEntries || 0), browserDroppedBodyBytes: Number(browserEvidence.droppedBodyBytes || 0) }, valuesRedacted: true };
|
||||
const artifacts = await realtimeWriteEvidenceArtifacts(reportDir, network, [], "existing-session-refresh");
|
||||
report.artifacts = artifacts;
|
||||
const reportArtifact = await realtimeWriteReport(reportDir, report, "existing-session-refresh-report");
|
||||
@@ -527,6 +553,7 @@ async function validateExistingSessionRefresh(command) {
|
||||
valuesRedacted: true,
|
||||
};
|
||||
} catch (error) {
|
||||
realtimeAggregateLast = { ...realtimeAggregateLast, status: "failed", counters: aggregateCounters, failure: errorSummary(error), valuesRedacted: true };
|
||||
const failure = realtimeExistingRefreshFailure(phase, error);
|
||||
if (!screenshot) {
|
||||
screenshot = await captureScreenshot("existing-session-refresh-failed-" + safeId(command.id))
|
||||
@@ -649,10 +676,17 @@ async function realtimeArmExistingSessionRefreshProbe(targetPage, profile) {
|
||||
let config;
|
||||
try { config = JSON.parse(raw); } catch { return; }
|
||||
const NativeEventSource = window.EventSource;
|
||||
const state = { sources: [], connected: [], businessEventCount: 0, failures: [], errors: [] };
|
||||
const state = { sources: [], connected: [], businessEventCount: 0, failures: [], errors: [], droppedEntries: 0, droppedBodyEntries: 0, droppedBodyBytes: 0 };
|
||||
const maxEntries = Math.max(1, Number(config.maxEntries || 1));
|
||||
const maxEventTextChars = Math.max(1, Number(config.maxEventTextChars || 1));
|
||||
const bodyBytes = (value) => new TextEncoder().encode(String(value || "")).byteLength;
|
||||
const pushBounded = (items, value) => {
|
||||
items.push(value);
|
||||
if (items.length > maxEntries) { state.droppedEntries += items.length - maxEntries; items.splice(0, items.length - maxEntries); }
|
||||
};
|
||||
window.__unideskExistingSessionRefresh = state;
|
||||
if (typeof NativeEventSource !== "function") {
|
||||
state.errors.push({ kind: "eventsource-unavailable" });
|
||||
pushBounded(state.errors, { kind: "eventsource-unavailable" });
|
||||
return;
|
||||
}
|
||||
class ObservedEventSource extends NativeEventSource {
|
||||
@@ -661,17 +695,23 @@ async function realtimeArmExistingSessionRefreshProbe(targetPage, profile) {
|
||||
let pathname = "";
|
||||
try { pathname = new URL(String(url), window.location.href).pathname; } catch {}
|
||||
if (pathname !== config.productEventsPath) return;
|
||||
state.sources.push({ url: String(url), opened: 0 });
|
||||
pushBounded(state.sources, { url: String(url), opened: 0 });
|
||||
const source = state.sources[state.sources.length - 1];
|
||||
this.addEventListener("open", () => { source.opened += 1; });
|
||||
this.addEventListener(config.productReadyEvent, (event) => {
|
||||
try { state.connected.push(JSON.parse(event.data)); } catch { state.errors.push({ kind: "connected-parse" }); }
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors, { kind: "connected-too-large", dataBytes: bytes }); return; }
|
||||
try { pushBounded(state.connected, JSON.parse(raw)); } catch { pushBounded(state.errors, { kind: "connected-parse" }); }
|
||||
});
|
||||
this.addEventListener(config.businessEvent, () => { state.businessEventCount += 1; });
|
||||
this.addEventListener("workbench.error", (event) => {
|
||||
try { state.failures.push(JSON.parse(event.data)); } catch { state.errors.push({ kind: "workbench-error-parse" }); }
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors, { kind: "workbench-error-too-large", dataBytes: bytes }); return; }
|
||||
try { pushBounded(state.failures, JSON.parse(raw)); } catch { pushBounded(state.errors, { kind: "workbench-error-parse" }); }
|
||||
});
|
||||
this.addEventListener("error", () => { state.errors.push({ kind: "eventsource-error" }); });
|
||||
this.addEventListener("error", () => { pushBounded(state.errors, { kind: "eventsource-error" }); });
|
||||
}
|
||||
}
|
||||
Object.defineProperty(ObservedEventSource, "CONNECTING", { value: NativeEventSource.CONNECTING });
|
||||
@@ -685,12 +725,16 @@ async function realtimeArmExistingSessionRefreshProbe(targetPage, profile) {
|
||||
productEventsPath: profile.productEventsPath,
|
||||
productReadyEvent: profile.productReadyEvent,
|
||||
businessEvent: profile.businessEvent,
|
||||
maxEntries: profile.aggregateBudget.perArrayEntries,
|
||||
maxEventTextChars: profile.aggregateBudget.perEventBodyBytes,
|
||||
});
|
||||
}
|
||||
|
||||
async function realtimeWorkbenchDomIdentity() {
|
||||
return page.evaluate(() => {
|
||||
const cards = Array.from(document.querySelectorAll("#conversation-list > .message-card"));
|
||||
return page.evaluate((maxRows) => {
|
||||
const cardNodes = document.querySelectorAll("#conversation-list > .message-card");
|
||||
if (cardNodes.length > maxRows) return { overflow: true, messageCount: cardNodes.length, maxRows, identities: [], missingIdentityRows: [], duplicateIdentities: [], traceIds: [], rows: [], valuesRedacted: true };
|
||||
const cards = Array.from(cardNodes);
|
||||
const rows = cards.map((element, index) => ({
|
||||
index,
|
||||
role: element.getAttribute("data-role") || null,
|
||||
@@ -709,10 +753,11 @@ async function realtimeWorkbenchDomIdentity() {
|
||||
rows,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
});
|
||||
}, maxDomEvidenceRows);
|
||||
}
|
||||
|
||||
function realtimeAssertSameWorkbenchDom(before, after) {
|
||||
if (before.overflow === true || after.overflow === true) throw new Error("Workbench DOM identity evidence exceeds the YAML row cap");
|
||||
if (before.missingIdentityRows.length > 0 || after.missingIdentityRows.length > 0) throw new Error("Workbench DOM contains rows without a stable message identity");
|
||||
if (before.duplicateIdentities.length > 0 || after.duplicateIdentities.length > 0) throw new Error("Workbench DOM contains duplicate message identities");
|
||||
if (before.messageCount !== after.messageCount) throw new Error("Workbench message count changed across refresh: " + before.messageCount + " -> " + after.messageCount);
|
||||
@@ -771,6 +816,21 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) {
|
||||
const terminalTimeoutMs = hasDurationOverride
|
||||
? Math.min(Number(profile.terminalTimeoutMs), Math.max(1000, Number(durationMs)))
|
||||
: Number(profile.terminalTimeoutMs);
|
||||
const entrySlots = 17;
|
||||
const eventStreamSlots = 8;
|
||||
const perArrayEntries = Math.max(1, Math.min(maxRealtimeEventEntries, Math.floor(maxRealtimeTotalEntries / entrySlots)));
|
||||
const bodyRetentionSlots = eventStreamSlots * (perArrayEntries + 1);
|
||||
const aggregateBudget = {
|
||||
maxTotalEntries: maxRealtimeTotalEntries,
|
||||
maxTotalBodyBytes: maxRealtimeTotalBodyBytes,
|
||||
entrySlots,
|
||||
eventStreamSlots,
|
||||
perArrayEntries,
|
||||
bodyRetentionSlots,
|
||||
perEventBodyBytes: Math.max(1, Math.min(maxRealtimeEventBodyBytes, Math.floor(maxRealtimeTotalBodyBytes / bodyRetentionSlots))),
|
||||
allocation: "partitioned-across-network-product-debug-active-arrays",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return {
|
||||
...profile,
|
||||
debugStreams,
|
||||
@@ -785,6 +845,7 @@ function realtimeFanoutEffectiveProfile(profile, durationMs) {
|
||||
conditionalEventTypePairs: profile.conditionalEventTypePairs && typeof profile.conditionalEventTypePairs === "object" ? sanitize(profile.conditionalEventTypePairs) : {},
|
||||
expectedProductSse: sanitize(expectedProductSse),
|
||||
expectedKafka: sanitize(expectedKafka),
|
||||
aggregateBudget,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -795,7 +856,14 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
|
||||
const key = label + "-" + sessionId;
|
||||
await subscriberPage.evaluate(({ key: streamKey, sessionId: scopedSessionId, profile: inputProfile }) => {
|
||||
const root = window.__unideskRealtimeFanoutProducts ??= {};
|
||||
const state = { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [], source: null };
|
||||
const state = { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [], source: null, droppedEntries: 0, droppedBodyEntries: 0, droppedBodyBytes: 0 };
|
||||
const maxEntries = Math.max(1, Number(inputProfile.maxEntries || 1));
|
||||
const maxEventTextChars = Math.max(1, Number(inputProfile.maxEventTextChars || 1));
|
||||
const bodyBytes = (value) => new TextEncoder().encode(String(value || "")).byteLength;
|
||||
const pushBounded = (items, value) => {
|
||||
items.push(value);
|
||||
if (items.length > maxEntries) { state.droppedEntries += items.length - maxEntries; items.splice(0, items.length - maxEntries); }
|
||||
};
|
||||
root[streamKey] = state;
|
||||
const firstText = (...values) => {
|
||||
for (const value of values) if (typeof value === "string" && value.trim()) return value.trim();
|
||||
@@ -811,12 +879,15 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
|
||||
state.source = source;
|
||||
source.onopen = () => { state.openCount += 1; };
|
||||
const record = (eventName, raw) => {
|
||||
const textRaw = String(raw || "");
|
||||
const bytes = bodyBytes(textRaw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors, { at: Date.now(), event: "event-too-large", dataBytes: bytes }); return; }
|
||||
let value;
|
||||
try { value = JSON.parse(raw); } catch { return; }
|
||||
try { value = JSON.parse(textRaw); } catch { return; }
|
||||
const nested = value?.event && typeof value.event === "object" ? value.event : {};
|
||||
const context = value?.context && typeof value.context === "object" ? value.context : {};
|
||||
const eventType = firstText(nested.eventType, nested.type, value.eventType, value.type);
|
||||
state.records.push({
|
||||
pushBounded(state.records, {
|
||||
eventName,
|
||||
at: Date.now(),
|
||||
schema: firstText(value.schema),
|
||||
@@ -831,19 +902,21 @@ async function realtimeNewSubscriber(storageState, sessionId, label, profile) {
|
||||
userMessageId: firstText(value.userMessageId, nested.userMessageId, nested.messageId),
|
||||
terminal: value.terminal === true || nested.terminal === true || eventType === "terminal" || eventType === "terminal_status",
|
||||
status: firstText(nested.status, value.status),
|
||||
envelopeFingerprint: fingerprint(raw),
|
||||
envelopeFingerprint: fingerprint(textRaw),
|
||||
valuesRedacted: true,
|
||||
});
|
||||
if (state.records.length > 800) state.records.splice(0, state.records.length - 800);
|
||||
};
|
||||
source.addEventListener(inputProfile.productReadyEvent, (event) => {
|
||||
state.connectedEventCount += 1;
|
||||
try { state.connected = JSON.parse(event.data); } catch { state.connected = { parseError: true }; }
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; state.connected = { tooLarge: true, dataBytes: bytes }; return; }
|
||||
try { state.connected = JSON.parse(raw); } catch { state.connected = { parseError: true }; }
|
||||
});
|
||||
source.addEventListener(inputProfile.businessEvent, (event) => record(inputProfile.businessEvent, event.data));
|
||||
source.addEventListener("workbench.error", (event) => state.errors.push({ at: Date.now(), event: "workbench.error", dataBytes: String(event.data || "").length }));
|
||||
source.onerror = () => state.errors.push({ at: Date.now(), event: "transport.error" });
|
||||
}, { key, sessionId, profile: { productEventsPath: profile.productEventsPath, productReadyEvent: profile.productReadyEvent, businessEvent: profile.businessEvent } });
|
||||
source.addEventListener("workbench.error", (event) => pushBounded(state.errors, { at: Date.now(), event: "workbench.error", dataBytes: String(event.data || "").length }));
|
||||
source.onerror = () => pushBounded(state.errors, { at: Date.now(), event: "transport.error" });
|
||||
}, { key, sessionId, profile: { productEventsPath: profile.productEventsPath, productReadyEvent: profile.productReadyEvent, businessEvent: profile.businessEvent, maxEntries: profile.aggregateBudget.perArrayEntries, maxEventTextChars: profile.aggregateBudget.perEventBodyBytes } });
|
||||
return { label, key, context: subscriberContext, page: subscriberPage };
|
||||
}
|
||||
|
||||
@@ -872,7 +945,7 @@ async function realtimeGotoHealth(targetPage, timeoutMs, profile, label) {
|
||||
wrapped.details = { ...(wrapped.details || {}), label, attempts, valuesRedacted: true };
|
||||
throw wrapped;
|
||||
}
|
||||
await sleep(profile.navigationRetryDelayMs * attempt);
|
||||
await interruptibleRunnerSleep(profile.navigationRetryDelayMs * attempt);
|
||||
}
|
||||
}
|
||||
throw new Error(label + " health navigation retry exhausted");
|
||||
@@ -943,8 +1016,8 @@ async function realtimeRunTurn(input) {
|
||||
if (!admissionOutcome.response) throw new Error("realtime turn admission response failed: " + String(admissionOutcome.error?.message || admissionOutcome.error || "unknown"));
|
||||
const admission = admissionOutcome.response;
|
||||
if (admission.status() < 200 || admission.status() >= 300) throw new Error("realtime turn admission returned HTTP " + admission.status());
|
||||
let admissionBody = {};
|
||||
try { admissionBody = await admission.json(); } catch {}
|
||||
const admissionBodyResult = await boundedResponseJson(admission);
|
||||
const admissionBody = admissionBodyResult.value || {};
|
||||
const canonicalTraceId = realtimeFirstText(admissionBody?.traceId, admissionBody?.result?.traceId, captured.traceId);
|
||||
if (canonicalTraceId !== captured.traceId) throw new Error("canonical trace changed after ready barrier");
|
||||
const nonTerminal = [];
|
||||
@@ -970,7 +1043,14 @@ async function realtimeRunTurn(input) {
|
||||
async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
|
||||
await debug.page.evaluate(({ key: streamKey, traceId: scopedTraceId, profile: inputProfile }) => {
|
||||
const root = window.__unideskRealtimeFanoutDebug ??= {};
|
||||
const state = { connected: {}, connectedEventCount: {}, openCount: {}, records: {}, errors: {}, sources: {} };
|
||||
const state = { connected: {}, connectedEventCount: {}, openCount: {}, records: {}, errors: {}, sources: {}, droppedEntries: 0, droppedBodyEntries: 0, droppedBodyBytes: 0 };
|
||||
const maxEntries = Math.max(1, Number(inputProfile.maxEntries || 1));
|
||||
const maxEventTextChars = Math.max(1, Number(inputProfile.maxEventTextChars || 1));
|
||||
const bodyBytes = (value) => new TextEncoder().encode(String(value || "")).byteLength;
|
||||
const pushBounded = (items, value) => {
|
||||
items.push(value);
|
||||
if (items.length > maxEntries) { state.droppedEntries += items.length - maxEntries; items.splice(0, items.length - maxEntries); }
|
||||
};
|
||||
root[streamKey] = state;
|
||||
const firstText = (...values) => {
|
||||
for (const value of values) if (typeof value === "string" && value.trim()) return value.trim();
|
||||
@@ -1001,11 +1081,17 @@ async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
|
||||
source.onopen = () => { state.openCount[stream] += 1; };
|
||||
source.addEventListener(inputProfile.debugReadyEvent, (event) => {
|
||||
state.connectedEventCount[stream] += 1;
|
||||
try { state.connected[stream] = JSON.parse(event.data); } catch { state.connected[stream] = { parseError: true }; }
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; state.connected[stream] = { tooLarge: true, dataBytes: bytes }; return; }
|
||||
try { state.connected[stream] = JSON.parse(raw); } catch { state.connected[stream] = { parseError: true }; }
|
||||
});
|
||||
source.addEventListener("hwlab.kafka.event", (event) => {
|
||||
const raw = String(event.data || "");
|
||||
const bytes = bodyBytes(raw);
|
||||
if (bytes > maxEventTextChars) { state.droppedBodyEntries += 1; state.droppedBodyBytes += bytes; pushBounded(state.errors[stream], { at: Date.now(), kind: "event-too-large", dataBytes: bytes }); return; }
|
||||
let outer;
|
||||
try { outer = JSON.parse(event.data); } catch { return; }
|
||||
try { outer = JSON.parse(raw); } catch { return; }
|
||||
const value = outer?.value && typeof outer.value === "object" ? outer.value : {};
|
||||
const nested = value?.event && typeof value.event === "object" ? value.event : {};
|
||||
const payload = nested?.payload && typeof nested.payload === "object" ? nested.payload : {};
|
||||
@@ -1017,7 +1103,7 @@ async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
|
||||
: stream === "hwlab"
|
||||
? firstText(nested.eventType, nested.type, value.eventType)
|
||||
: firstText(value.eventType, value.stdio?.method);
|
||||
state.records[stream].push({
|
||||
pushBounded(state.records[stream], {
|
||||
stream,
|
||||
at: Date.now(),
|
||||
topic: firstText(outer.topic),
|
||||
@@ -1042,16 +1128,17 @@ async function realtimeOpenDebugStreams(debug, key, traceId, profile) {
|
||||
envelopeFingerprint: stream === "hwlab" ? fingerprint(JSON.stringify(value)) : null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
if (state.records[stream].length > 1000) state.records[stream].splice(0, state.records[stream].length - 1000);
|
||||
});
|
||||
source.addEventListener("hwlab.kafka.error", (event) => state.errors[stream].push({ at: Date.now(), dataBytes: String(event.data || "").length }));
|
||||
source.onerror = () => state.errors[stream].push({ at: Date.now(), transport: true });
|
||||
source.addEventListener("hwlab.kafka.error", (event) => pushBounded(state.errors[stream], { at: Date.now(), dataBytes: String(event.data || "").length }));
|
||||
source.onerror = () => pushBounded(state.errors[stream], { at: Date.now(), transport: true });
|
||||
}
|
||||
}, { key, traceId, profile: {
|
||||
debugStreams: profile.debugStreams,
|
||||
debugEventsPath: profile.debugEventsPath,
|
||||
debugReadyEvent: profile.debugReadyEvent,
|
||||
fromBeginning: profile.fromBeginning,
|
||||
maxEntries: profile.aggregateBudget.perArrayEntries,
|
||||
maxEventTextChars: profile.aggregateBudget.perEventBodyBytes,
|
||||
} });
|
||||
}
|
||||
|
||||
@@ -1143,7 +1230,7 @@ async function realtimeWaitTurnStable(debug, key, traceId, subscribers, profile)
|
||||
} else if (Date.now() - stableSince >= profile.terminalQuietMs) {
|
||||
return { debug: debugSnapshot, products: productSnapshots };
|
||||
}
|
||||
await sleep(250);
|
||||
await interruptibleRunnerSleep(250);
|
||||
}
|
||||
throw new Error("realtime debug/product streams did not become quiet after terminal");
|
||||
}
|
||||
@@ -1169,7 +1256,7 @@ async function realtimeProductSnapshot(subscriber) {
|
||||
return subscriber.page.evaluate((key) => {
|
||||
const state = window.__unideskRealtimeFanoutProducts?.[key];
|
||||
if (!state) return { connected: null, connectedEventCount: 0, openCount: 0, records: [], errors: [{ missing: true }] };
|
||||
return { connected: state.connected ? JSON.parse(JSON.stringify(state.connected)) : null, connectedEventCount: state.connectedEventCount, openCount: state.openCount, records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)) };
|
||||
return { connected: state.connected ? JSON.parse(JSON.stringify(state.connected)) : null, connectedEventCount: state.connectedEventCount, openCount: state.openCount, records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)), droppedEntries: Number(state.droppedEntries || 0), droppedBodyEntries: Number(state.droppedBodyEntries || 0), droppedBodyBytes: Number(state.droppedBodyBytes || 0) };
|
||||
}, subscriber.key);
|
||||
}
|
||||
|
||||
@@ -1177,7 +1264,7 @@ async function realtimeDebugSnapshot(debug, key) {
|
||||
return debug.page.evaluate((streamKey) => {
|
||||
const state = window.__unideskRealtimeFanoutDebug?.[streamKey];
|
||||
if (!state) return { connected: {}, connectedEventCount: {}, openCount: {}, records: { stdio: [], agentrun: [], hwlab: [] }, errors: { stdio: [], agentrun: [], hwlab: [] } };
|
||||
return { connected: JSON.parse(JSON.stringify(state.connected)), connectedEventCount: JSON.parse(JSON.stringify(state.connectedEventCount)), openCount: JSON.parse(JSON.stringify(state.openCount)), records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)) };
|
||||
return { connected: JSON.parse(JSON.stringify(state.connected)), connectedEventCount: JSON.parse(JSON.stringify(state.connectedEventCount)), openCount: JSON.parse(JSON.stringify(state.openCount)), records: JSON.parse(JSON.stringify(state.records)), errors: JSON.parse(JSON.stringify(state.errors)), droppedEntries: Number(state.droppedEntries || 0), droppedBodyEntries: Number(state.droppedBodyEntries || 0), droppedBodyBytes: Number(state.droppedBodyBytes || 0) };
|
||||
}, key);
|
||||
}
|
||||
|
||||
@@ -1191,6 +1278,9 @@ async function realtimeAllDebugSnapshots(debug) {
|
||||
openCount: JSON.parse(JSON.stringify(state.openCount || {})),
|
||||
records: JSON.parse(JSON.stringify(state.records || {})),
|
||||
errors: JSON.parse(JSON.stringify(state.errors || {})),
|
||||
droppedEntries: Number(state.droppedEntries || 0),
|
||||
droppedBodyEntries: Number(state.droppedBodyEntries || 0),
|
||||
droppedBodyBytes: Number(state.droppedBodyBytes || 0),
|
||||
}));
|
||||
});
|
||||
}
|
||||
@@ -1282,7 +1372,7 @@ async function realtimeWaitReplayedTraceJointQuiet(debug, key, subscriber, trace
|
||||
if (comparison.equal && terminalSeen && sourceTransportStable && quietForMs >= profile.terminalQuietMs) {
|
||||
return { debug: debugSnapshot, product: productSnapshot, comparison, quietForMs, valuesRedacted: true };
|
||||
}
|
||||
await sleep(250);
|
||||
await interruptibleRunnerSleep(250);
|
||||
}
|
||||
const error = new Error(label + " Kafka source and product replay envelope multisets did not converge before the joint quiet timeout");
|
||||
error.details = {
|
||||
@@ -1589,6 +1679,8 @@ function realtimeTurnSummary(turn, debug, product, terminal) {
|
||||
}
|
||||
|
||||
async function realtimeCaptureSubscriberScreenshot(targetPage, reportDir, name, summary) {
|
||||
const screenshotSkipped = reserveScreenshot("realtime-fanout-" + name);
|
||||
if (screenshotSkipped !== null) return screenshotSkipped;
|
||||
await targetPage.evaluate((value) => {
|
||||
document.documentElement.style.background = "#0b1020";
|
||||
document.body.innerHTML = "";
|
||||
@@ -1598,7 +1690,7 @@ async function realtimeCaptureSubscriberScreenshot(targetPage, reportDir, name,
|
||||
document.body.appendChild(pre);
|
||||
}, summary);
|
||||
const file = path.join(reportDir, name);
|
||||
await targetPage.screenshot({ path: file, fullPage: true, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
|
||||
await targetPage.screenshot({ path: file, fullPage: false, animations: "disabled", timeout: screenshotCaptureTimeoutMs });
|
||||
const meta = await fileMeta(file);
|
||||
const artifact = { kind: "realtime-fanout-screenshot", path: file, name, byteCount: meta.byteCount, sha256: meta.sha256, commandId: activeCommandId, valuesRedacted: true };
|
||||
await appendJsonl(files.artifacts, { ts: new Date().toISOString(), ...artifact });
|
||||
@@ -1679,7 +1771,7 @@ async function realtimePoll(read, timeoutMs, label) {
|
||||
while (Date.now() < deadline) {
|
||||
const value = await read();
|
||||
if (value) return value;
|
||||
await sleep(250);
|
||||
await interruptibleRunnerSleep(250);
|
||||
}
|
||||
throw new Error(label + " timed out after " + timeoutMs + "ms");
|
||||
}
|
||||
|
||||
@@ -73,14 +73,34 @@ async function writeManifest(extra = {}) {
|
||||
maxPerformanceEntries,
|
||||
maxBrowserProcessHistoryEntries,
|
||||
maxBrowserFreezeSignalHistoryEntries,
|
||||
maxRealtimeEventEntries,
|
||||
maxRealtimeEventBodyBytes,
|
||||
maxDomEvidenceRows,
|
||||
maxProcessScanEntries,
|
||||
maxPerformanceCaptureMs,
|
||||
responseBodyMaxBytes,
|
||||
webPerformanceRequestBodyMaxBytes,
|
||||
maxJsonlQueueEntries,
|
||||
maxJsonlQueueBytes,
|
||||
maxJsonlLineBytes,
|
||||
jsonlFlushTimeoutMs,
|
||||
maxResponseBodyInFlight,
|
||||
maxPassiveListenerTasks,
|
||||
passiveTaskFlushTimeoutMs,
|
||||
closeStepTimeoutMs,
|
||||
maxProcessReadConcurrency,
|
||||
maxRealtimeTotalEntries,
|
||||
maxRealtimeTotalBodyBytes,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
jsonlRotation,
|
||||
jsonlWriter: jsonlWriterStats(),
|
||||
responseBodies: responseBodyStats(),
|
||||
passiveTasks: passiveTaskStats(),
|
||||
realtimeAggregate: realtimeAggregateLast,
|
||||
commandDirs: dirs,
|
||||
artifacts: files,
|
||||
safety: { pureClient: true, inboundApi: false, database: false, queueConsumer: false, k8sWorkload: false, valuesRedacted: true, secretValuesPrinted: false },
|
||||
@@ -115,8 +135,24 @@ async function writeHeartbeat(extra = {}) {
|
||||
maxPerformanceEntries,
|
||||
maxBrowserProcessHistoryEntries,
|
||||
maxBrowserFreezeSignalHistoryEntries,
|
||||
maxRealtimeEventEntries,
|
||||
maxRealtimeEventBodyBytes,
|
||||
maxDomEvidenceRows,
|
||||
maxProcessScanEntries,
|
||||
maxPerformanceCaptureMs,
|
||||
responseBodyMaxBytes,
|
||||
webPerformanceRequestBodyMaxBytes,
|
||||
maxJsonlQueueEntries,
|
||||
maxJsonlQueueBytes,
|
||||
maxJsonlLineBytes,
|
||||
jsonlFlushTimeoutMs,
|
||||
maxResponseBodyInFlight,
|
||||
maxPassiveListenerTasks,
|
||||
passiveTaskFlushTimeoutMs,
|
||||
closeStepTimeoutMs,
|
||||
maxProcessReadConcurrency,
|
||||
maxRealtimeTotalEntries,
|
||||
maxRealtimeTotalBodyBytes,
|
||||
valuesRedacted: true,
|
||||
},
|
||||
lastObserverRefreshAt: Number.isFinite(lastObserverRefreshAtMs) ? new Date(lastObserverRefreshAtMs).toISOString() : null,
|
||||
@@ -124,6 +160,10 @@ async function writeHeartbeat(extra = {}) {
|
||||
sampleSeq,
|
||||
commandSeq,
|
||||
activeCommandId,
|
||||
jsonlWriter: jsonlWriterStats(),
|
||||
responseBodies: responseBodyStats(),
|
||||
passiveTasks: passiveTaskStats(),
|
||||
realtimeAggregate: realtimeAggregateLast,
|
||||
updatedAt: new Date().toISOString(),
|
||||
uptimeMs: Date.now() - startedAtMs,
|
||||
...extra,
|
||||
@@ -131,6 +171,22 @@ async function writeHeartbeat(extra = {}) {
|
||||
await writeFile(files.heartbeat, JSON.stringify(heartbeat, null, 2) + "\n", { mode: 0o600 });
|
||||
}
|
||||
|
||||
async function writeTerminalResourceStats(extra = {}) {
|
||||
for (const file of [files.manifest, files.heartbeat]) {
|
||||
let existing = {};
|
||||
try { existing = JSON.parse(await readFile(file, "utf8")); } catch {}
|
||||
await writeFile(file, JSON.stringify({
|
||||
...existing,
|
||||
jsonlWriter: jsonlWriterStats(),
|
||||
responseBodies: responseBodyStats(),
|
||||
passiveTasks: passiveTaskStats(),
|
||||
terminalResources: sanitize(extra),
|
||||
updatedAt: new Date().toISOString(),
|
||||
valuesRedacted: true,
|
||||
}, null, 2) + "\n", { mode: 0o600 });
|
||||
}
|
||||
}
|
||||
|
||||
function startHeartbeatPulse() {
|
||||
const timer = setInterval(() => {
|
||||
if (stopping) return;
|
||||
@@ -410,62 +466,96 @@ async function triggerBrowserFreezeBlocker(trigger) {
|
||||
}
|
||||
|
||||
async function terminateBrowserForFreezeBlocker(blocker) {
|
||||
const childProcess = browser && typeof browser.process === "function" ? browser.process() : null;
|
||||
const pid = Number(childProcess?.pid);
|
||||
if (!Number.isFinite(pid) || pid <= 0) {
|
||||
return { ok: false, reason: "browser-process-unavailable", blockerKind: blocker.kind, valuesRedacted: true };
|
||||
const result = await terminateTrackedBrowserProcesses("freeze-blocker:" + String(blocker.kind || "unknown"));
|
||||
return { ...result, blockerKind: blocker.kind, valuesRedacted: true };
|
||||
}
|
||||
|
||||
function rememberTrackedBrowserProcesses(processes) {
|
||||
const merged = new Map(trackedBrowserProcessIdentities.map((item) => [item.pid + ":" + item.startTicks, item]));
|
||||
for (const item of Array.isArray(processes) ? processes : []) {
|
||||
if (!Number.isInteger(item.pid) || !Number.isInteger(item.startTicks) || !item.role) continue;
|
||||
merged.set(item.pid + ":" + item.startTicks, {
|
||||
pid: item.pid,
|
||||
ppid: item.ppid,
|
||||
pgid: item.pgid,
|
||||
sid: item.sid,
|
||||
startTicks: item.startTicks,
|
||||
role: item.role,
|
||||
commandHash: item.commandHash || null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
trackedBrowserProcessIdentities = [...merged.values()].slice(-Math.min(maxProcessScanEntries, 1024));
|
||||
}
|
||||
|
||||
async function matchingTrackedBrowserProcesses() {
|
||||
const rows = await Promise.all(trackedBrowserProcessIdentities.map(async (identity) => {
|
||||
const current = await readOneProcProcessStat(identity.pid).catch(() => null);
|
||||
return current && current.startTicks === identity.startTicks ? { ...identity, currentPpid: current.ppid, currentPgid: current.pgid, currentSid: current.sid } : null;
|
||||
}));
|
||||
return rows.filter(Boolean);
|
||||
}
|
||||
|
||||
async function waitForTrackedBrowserExit(timeoutMs, pollIntervalMs) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let remaining = await matchingTrackedBrowserProcesses();
|
||||
while (remaining.length > 0 && Date.now() <= deadline) {
|
||||
await boundedCleanupSleep(pollIntervalMs);
|
||||
remaining = await matchingTrackedBrowserProcesses();
|
||||
}
|
||||
return remaining;
|
||||
}
|
||||
|
||||
async function terminateTrackedBrowserProcesses(reason, requiredDiscovery = null) {
|
||||
let refreshDiscovery;
|
||||
try {
|
||||
const summary = await collectChromiumProcessSummary(null);
|
||||
refreshDiscovery = { ok: true, processScan: summary.processScan, chromiumProcessCount: summary.chromiumProcessCount, valuesRedacted: true };
|
||||
} catch (error) {
|
||||
refreshDiscovery = { ok: false, processScan: lastProcessScan, error: errorSummary(error), valuesRedacted: true };
|
||||
}
|
||||
const discovery = requiredDiscovery || refreshDiscovery;
|
||||
const identityScanUnavailable = browserLaunchAttempted === true
|
||||
&& (discovery?.ok !== true || discovery?.processScan?.complete !== true);
|
||||
const initial = await matchingTrackedBrowserProcesses();
|
||||
const result = {
|
||||
ok: false,
|
||||
pid: Math.floor(pid),
|
||||
ok: initial.length === 0 && !identityScanUnavailable,
|
||||
reason,
|
||||
browserLaunchAttempted,
|
||||
identityScanUnavailable,
|
||||
requiredDiscovery: discovery,
|
||||
refreshDiscovery,
|
||||
trackedIdentityCount: trackedBrowserProcessIdentities.length,
|
||||
initialLiveCount: initial.length,
|
||||
gracefulSignal: browserFreezePolicy.kill.gracefulSignal,
|
||||
forceSignal: browserFreezePolicy.kill.forceSignal,
|
||||
graceMs: browserFreezePolicy.kill.graceMs,
|
||||
pollIntervalMs: browserFreezePolicy.kill.pollIntervalMs,
|
||||
gracefulSent: false,
|
||||
forceSent: false,
|
||||
exitedAfterGrace: false,
|
||||
exitedAfterForce: false,
|
||||
gracefulSentCount: 0,
|
||||
forceSentCount: 0,
|
||||
remainingIdentityCount: initial.length,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
try {
|
||||
childProcess.kill(browserFreezePolicy.kill.gracefulSignal);
|
||||
result.gracefulSent = true;
|
||||
} catch (error) {
|
||||
result.gracefulError = errorSummary(error);
|
||||
}
|
||||
result.exitedAfterGrace = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
if (!result.exitedAfterGrace) {
|
||||
for (const item of [...initial].sort((a, b) => b.pid - a.pid)) {
|
||||
try {
|
||||
childProcess.kill(browserFreezePolicy.kill.forceSignal);
|
||||
result.forceSent = true;
|
||||
} catch (error) {
|
||||
result.forceError = errorSummary(error);
|
||||
}
|
||||
result.exitedAfterForce = await waitForPidExit(result.pid, browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
process.kill(item.pid, browserFreezePolicy.kill.gracefulSignal);
|
||||
result.gracefulSentCount += 1;
|
||||
} catch {}
|
||||
}
|
||||
result.ok = result.exitedAfterGrace || result.exitedAfterForce;
|
||||
let remaining = await waitForTrackedBrowserExit(browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
for (const item of [...remaining].sort((a, b) => b.pid - a.pid)) {
|
||||
try {
|
||||
process.kill(item.pid, browserFreezePolicy.kill.forceSignal);
|
||||
result.forceSentCount += 1;
|
||||
} catch {}
|
||||
}
|
||||
remaining = await waitForTrackedBrowserExit(browserFreezePolicy.kill.graceMs, browserFreezePolicy.kill.pollIntervalMs);
|
||||
result.remainingIdentityCount = remaining.length;
|
||||
result.remainingIdentities = remaining.slice(0, 50).map((item) => ({ pid: item.pid, startTicks: item.startTicks, role: item.role, originalPpid: item.ppid, currentPpid: item.currentPpid, originalSid: item.sid, currentSid: item.currentSid, valuesRedacted: true }));
|
||||
result.ok = remaining.length === 0 && !identityScanUnavailable;
|
||||
return result;
|
||||
}
|
||||
|
||||
async function waitForPidExit(pid, timeoutMs, pollIntervalMs) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() <= deadline) {
|
||||
if (!pidAlive(pid)) return true;
|
||||
await sleep(pollIntervalMs);
|
||||
}
|
||||
return !pidAlive(pid);
|
||||
}
|
||||
|
||||
function pidAlive(pid) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function browserProcessSampleRef(sample) {
|
||||
return {
|
||||
ts: sample?.ts ?? null,
|
||||
@@ -532,13 +622,16 @@ async function collectChromiumProcessSummary(browserPid) {
|
||||
queue.push(child);
|
||||
}
|
||||
}
|
||||
const processes = all
|
||||
.filter((item) => descendants.has(item.pid))
|
||||
const detailed = await readProcProcessDetails(all.filter((item) => descendants.has(item.pid)));
|
||||
const processes = detailed
|
||||
.map((item) => ({ ...item, role: classifyChromiumProcess(item) }))
|
||||
.filter((item) => item.role)
|
||||
.map((item) => ({
|
||||
pid: item.pid,
|
||||
ppid: item.ppid,
|
||||
pgid: item.pgid,
|
||||
sid: item.sid,
|
||||
startTicks: item.startTicks,
|
||||
name: item.name || null,
|
||||
role: item.role,
|
||||
rssBytes: item.rssBytes,
|
||||
@@ -548,6 +641,7 @@ async function collectChromiumProcessSummary(browserPid) {
|
||||
valuesRedacted: true,
|
||||
}))
|
||||
.sort((a, b) => Number(b.rssBytes || 0) - Number(a.rssBytes || 0));
|
||||
rememberTrackedBrowserProcesses(processes);
|
||||
const roles = {};
|
||||
for (const item of processes) {
|
||||
const role = item.role || "unknown";
|
||||
@@ -561,6 +655,7 @@ async function collectChromiumProcessSummary(browserPid) {
|
||||
const maxProcess = processes[0] || null;
|
||||
return {
|
||||
sampledRootPids: roots,
|
||||
processScan: lastProcessScan,
|
||||
chromiumProcessCount: processes.length,
|
||||
totalRssBytes,
|
||||
totalRssMb: bytesToMb(totalRssBytes),
|
||||
@@ -577,28 +672,68 @@ async function readProcProcessTable() {
|
||||
let entries = [];
|
||||
try {
|
||||
entries = await readdir("/proc");
|
||||
} catch {
|
||||
return [];
|
||||
} catch (error) {
|
||||
lastProcessScan = { ok: false, complete: false, totalPidEntries: 0, scannedPidEntries: 0, failedPidEntries: 0, truncated: false, readConcurrency: maxProcessReadConcurrency, error: errorSummary(error), valuesRedacted: true };
|
||||
throw new Error("proc-process-table-unavailable");
|
||||
}
|
||||
const numeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0);
|
||||
const rows = await Promise.all(numeric.map((pid) => readOneProcProcess(pid).catch(() => null)));
|
||||
return rows.filter(Boolean);
|
||||
const allNumeric = entries.map((name) => Number(name)).filter((pid) => Number.isInteger(pid) && pid > 0).sort((a, b) => b - a);
|
||||
const numeric = allNumeric.slice(0, maxProcessScanEntries);
|
||||
const rows = [];
|
||||
for (let offset = 0; offset < numeric.length; offset += maxProcessReadConcurrency) {
|
||||
const batch = await Promise.all(numeric.slice(offset, offset + maxProcessReadConcurrency).map((pid) => readOneProcProcessStat(pid).catch(() => null)));
|
||||
rows.push(...batch.filter(Boolean));
|
||||
}
|
||||
const failedPidEntries = numeric.length - rows.length;
|
||||
const truncated = numeric.length < allNumeric.length;
|
||||
const unusable = numeric.length > 0 && rows.length === 0;
|
||||
lastProcessScan = {
|
||||
ok: !unusable,
|
||||
complete: !unusable && !truncated,
|
||||
totalPidEntries: allNumeric.length,
|
||||
scannedPidEntries: numeric.length,
|
||||
readablePidEntries: rows.length,
|
||||
failedPidEntries,
|
||||
truncated,
|
||||
readConcurrency: maxProcessReadConcurrency,
|
||||
error: unusable ? "all-proc-stat-reads-failed" : null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
if (unusable) throw new Error("proc-process-table-unreadable");
|
||||
return rows;
|
||||
}
|
||||
|
||||
async function readOneProcProcess(pid) {
|
||||
const [statText, statusText, cmdlineText] = await Promise.all([
|
||||
readFile(path.join("/proc", String(pid), "stat"), "utf8").catch(() => ""),
|
||||
async function readOneProcProcessStat(pid) {
|
||||
const statText = await readFile(path.join("/proc", String(pid), "stat"), "utf8");
|
||||
if (!statText) return null;
|
||||
return { pid, ...procIdentityFromStat(statText) };
|
||||
}
|
||||
|
||||
async function readProcProcessDetails(rows) {
|
||||
const detailed = [];
|
||||
for (let offset = 0; offset < rows.length; offset += maxProcessReadConcurrency) {
|
||||
const batch = await Promise.all(rows.slice(offset, offset + maxProcessReadConcurrency).map((row) => readOneProcProcessDetails(row).catch(() => null)));
|
||||
detailed.push(...batch.filter(Boolean));
|
||||
}
|
||||
return detailed;
|
||||
}
|
||||
|
||||
async function readOneProcProcessDetails(row) {
|
||||
const pid = row.pid;
|
||||
const [statusText, cmdlineText] = await Promise.all([
|
||||
readFile(path.join("/proc", String(pid), "status"), "utf8").catch(() => ""),
|
||||
readFile(path.join("/proc", String(pid), "cmdline"), "utf8").catch(() => ""),
|
||||
]);
|
||||
if (!statText && !statusText) return null;
|
||||
const ppid = procPpidFromStat(statText);
|
||||
if (!statusText && !cmdlineText) return null;
|
||||
const ppid = row.ppid;
|
||||
const name = procStatusField(statusText, "Name") || null;
|
||||
const rssKb = procStatusKb(statusText, "VmRSS");
|
||||
const vmSizeKb = procStatusKb(statusText, "VmSize");
|
||||
return {
|
||||
pid,
|
||||
ppid,
|
||||
pgid: row.pgid,
|
||||
sid: row.sid,
|
||||
startTicks: row.startTicks,
|
||||
name,
|
||||
cmdline: String(cmdlineText || "").replace(/\0/gu, " ").replace(/\s+/gu, " ").trim(),
|
||||
rssBytes: Number.isFinite(rssKb) ? rssKb * 1024 : 0,
|
||||
@@ -606,13 +741,13 @@ async function readOneProcProcess(pid) {
|
||||
};
|
||||
}
|
||||
|
||||
function procPpidFromStat(value) {
|
||||
function procIdentityFromStat(value) {
|
||||
const text = String(value || "");
|
||||
const end = text.lastIndexOf(")");
|
||||
if (end < 0) return null;
|
||||
if (end < 0) return { ppid: null, pgid: null, sid: null, startTicks: null };
|
||||
const parts = text.slice(end + 1).trim().split(/\s+/u);
|
||||
const ppid = Number(parts[1]);
|
||||
return Number.isFinite(ppid) ? ppid : null;
|
||||
const integerAt = (index) => Number.isInteger(Number(parts[index])) ? Number(parts[index]) : null;
|
||||
return { ppid: integerAt(1), pgid: integerAt(2), sid: integerAt(3), startTicks: integerAt(19) };
|
||||
}
|
||||
|
||||
function procStatusField(value, key) {
|
||||
@@ -725,7 +860,13 @@ function applyBrowserPageRuntimeBaseline(result) {
|
||||
const key = [result.pageRole || "unknown", result.pageId || "unknown", Number.isFinite(Number(result.pageEpoch)) ? Number(result.pageEpoch) : 0].join(":");
|
||||
const current = browserPageRuntimeMemorySnapshot(result);
|
||||
const existing = browserPageRuntimeBaselines.get(key);
|
||||
if (!existing && current) browserPageRuntimeBaselines.set(key, { ...current, capturedAt: new Date().toISOString(), source: "first-page-runtime-sample", valuesRedacted: true });
|
||||
if (!existing && current) {
|
||||
const prefix = [result.pageRole || "unknown", result.pageId || "unknown"].join(":") + ":";
|
||||
for (const existingKey of browserPageRuntimeBaselines.keys()) {
|
||||
if (existingKey.startsWith(prefix)) browserPageRuntimeBaselines.delete(existingKey);
|
||||
}
|
||||
browserPageRuntimeBaselines.set(key, { ...current, capturedAt: new Date().toISOString(), source: "first-page-runtime-sample", valuesRedacted: true });
|
||||
}
|
||||
const baseline = browserPageRuntimeBaselines.get(key) || null;
|
||||
result.baseline = baseline ? {
|
||||
capturedAt: baseline.capturedAt,
|
||||
@@ -847,8 +988,10 @@ function bytesToMb(value) {
|
||||
}
|
||||
|
||||
function attachPassiveListeners(targetPage, pageRole = "control", targetPageId = pageId) {
|
||||
void installPagePerformanceProbe(targetPage, pageRole, targetPageId)
|
||||
.catch((error) => appendJsonl(files.errors, eventRecord("performance-probe-install-error", { pageRole, pageId: targetPageId, error: errorSummary(error), valuesRedacted: true })));
|
||||
trackPassiveTask(
|
||||
() => installPagePerformanceProbe(targetPage, pageRole, targetPageId),
|
||||
"performance-probe-install:" + pageRole,
|
||||
);
|
||||
targetPage.on("request", (request) => {
|
||||
const webPerformancePayload = summarizeWebPerformanceRequestPayload(request);
|
||||
void appendJsonl(files.network, eventRecord("request", {
|
||||
@@ -878,19 +1021,10 @@ function attachPassiveListeners(targetPage, pageRole = "control", targetPageId =
|
||||
statusText: response.statusText(),
|
||||
fromServiceWorker: response.fromServiceWorker(),
|
||||
};
|
||||
void (async () => {
|
||||
trackPassiveTask(async () => {
|
||||
const bodyFields = await summarizeWorkbenchResponseBody(response, request);
|
||||
await appendJsonl(files.network, eventRecord("response", { ...base, ...bodyFields }));
|
||||
})().catch((error) => appendJsonl(files.errors, eventRecord("response-body-summary-error", {
|
||||
pageRole,
|
||||
pageId: targetPageId,
|
||||
sampleSeq,
|
||||
commandId: activeCommandId,
|
||||
method: request.method(),
|
||||
url: safeUrl(response.url()),
|
||||
error: errorSummary(error),
|
||||
valuesRedacted: true
|
||||
})));
|
||||
}, "response-body-summary:" + pageRole);
|
||||
});
|
||||
targetPage.on("requestfailed", (request) => {
|
||||
void appendJsonl(files.network, eventRecord("requestfailed", {
|
||||
|
||||
@@ -795,8 +795,7 @@ function digestSessionRail(value) {
|
||||
};
|
||||
}
|
||||
|
||||
async function captureScreenshot(reason, imageType = "png") {
|
||||
if (!page || page.isClosed()) throw new Error("page is not available for screenshot");
|
||||
function reserveScreenshot(reason) {
|
||||
if (screenshotCount >= maxScreenshots) {
|
||||
lastScreenshotAtMs = Date.now();
|
||||
return {
|
||||
@@ -814,6 +813,14 @@ async function captureScreenshot(reason, imageType = "png") {
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
screenshotCount += 1;
|
||||
return null;
|
||||
}
|
||||
|
||||
async function captureScreenshot(reason, imageType = "png") {
|
||||
if (!page || page.isClosed()) throw new Error("page is not available for screenshot");
|
||||
const skipped = reserveScreenshot(reason);
|
||||
if (skipped !== null) return skipped;
|
||||
if (screenshotCaptureState && screenshotCaptureState.settled !== true) {
|
||||
const ageMs = Date.now() - Number(screenshotCaptureState.startedAtMs || Date.now());
|
||||
const error = new Error("screenshot capture already in progress");
|
||||
@@ -831,7 +838,6 @@ async function captureScreenshot(reason, imageType = "png") {
|
||||
lastScreenshotAtMs = Date.now();
|
||||
throw error;
|
||||
}
|
||||
screenshotCount += 1;
|
||||
artifactSeq += 1;
|
||||
const safeReason = safeId(String(reason || "manual")).slice(0, 40) || "manual";
|
||||
const type = imageType === "jpeg" || imageType === "jpg" ? "jpeg" : "png";
|
||||
@@ -905,10 +911,10 @@ async function summarizeWorkbenchResponseBody(response, request) {
|
||||
const headers = response.headers();
|
||||
const contentType = String(headers["content-type"] || headers["Content-Type"] || "");
|
||||
if (!/json/iu.test(contentType)) return { bodyRead: false, bodyReadSkipped: "non-json", valuesRedacted: true };
|
||||
const contentLength = Number(headers["content-length"] || headers["Content-Length"]);
|
||||
const maxBytes = responseBodyMaxBytes;
|
||||
if (Number.isFinite(contentLength) && contentLength > maxBytes) return { bodyRead: false, bodyReadSkipped: "content-length-too-large", bodyByteCount: contentLength, valuesRedacted: true };
|
||||
const text = await response.text();
|
||||
const body = await boundedResponseText(response, maxBytes);
|
||||
if (body.ok !== true || typeof body.value !== "string") return { bodyRead: false, bodyReadSkipped: body.error?.reason || "bounded-response-read-failed", bodyContract: body.bodyContract, valuesRedacted: true };
|
||||
const text = body.value;
|
||||
const byteCount = Buffer.byteLength(text);
|
||||
if (byteCount > maxBytes) return { bodyRead: true, bodyReadSkipped: "body-too-large", bodyByteCount: byteCount, bodyHash: sha256Text(text), valuesRedacted: true };
|
||||
let parsed = null;
|
||||
|
||||
@@ -15,6 +15,7 @@ import { nodeWebObserveRunnerTraceReadabilitySource } from "./hwlab-node-web-obs
|
||||
export function nodeWebObserveRunnerSource(): string {
|
||||
return String.raw`#!/usr/bin/env node
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { appendFile, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
@@ -31,14 +32,30 @@ const targetPath = process.env.UNIDESK_WEB_OBSERVE_TARGET_PATH || "/workbench";
|
||||
const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS, 5000);
|
||||
const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000);
|
||||
const screenshotCaptureTimeoutMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_CAPTURE_TIMEOUT_MS, 15000, 1000, 120000);
|
||||
const maxSamples = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 720, 1, 10000000);
|
||||
const maxSamples = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, "UNIDESK_WEB_OBSERVE_MAX_SAMPLES", 1, 10000000);
|
||||
const observerRefreshIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS, 180000);
|
||||
const maxRunMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, 3600000, 1000, 86400000);
|
||||
const maxScreenshots = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS, 48, 1, 10000);
|
||||
const maxPerformanceEntries = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES, 2000, 1, 100000);
|
||||
const maxBrowserProcessHistoryEntries = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES, 360, 1, 100000);
|
||||
const maxBrowserFreezeSignalHistoryEntries = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES, 120, 1, 100000);
|
||||
const responseBodyMaxBytes = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES, 524288, 1024, 16777216);
|
||||
const maxRunMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RUN_MS, "UNIDESK_WEB_OBSERVE_MAX_RUN_MS", 1000, 86400000);
|
||||
const maxScreenshots = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS, "UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS", 1, 10000);
|
||||
const maxPerformanceEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES", 1, 100000);
|
||||
const maxBrowserProcessHistoryEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES", 1, 100000);
|
||||
const maxBrowserFreezeSignalHistoryEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES", 1, 100000);
|
||||
const maxRealtimeEventEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES", 1, 100000);
|
||||
const maxRealtimeEventBodyBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES", 1, 16777216);
|
||||
const maxDomEvidenceRows = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS, "UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS", 1, 100000);
|
||||
const maxProcessScanEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES", 1, 1000000);
|
||||
const maxPerformanceCaptureMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS, "UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS", 1, 600) * 1000;
|
||||
const responseBodyMaxBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES, "UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES", 1024, 16777216);
|
||||
const maxJsonlQueueEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES", 1, 100000);
|
||||
const maxJsonlQueueBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES, "UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES", 1024, 67108864);
|
||||
const maxJsonlLineBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES, "UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES", 1024, 16777216);
|
||||
const jsonlFlushTimeoutMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS, "UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS", 100, 60000);
|
||||
const maxResponseBodyInFlight = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT, "UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT", 1, 64);
|
||||
const maxPassiveListenerTasks = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS, "UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS", 1, 10000);
|
||||
const passiveTaskFlushTimeoutMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS, "UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS", 100, 60000);
|
||||
const closeStepTimeoutMs = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS, "UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS", 100, 60000);
|
||||
const maxProcessReadConcurrency = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY, "UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY", 1, 256);
|
||||
const maxRealtimeTotalEntries = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES", 1, 100000);
|
||||
const maxRealtimeTotalBodyBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES, "UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES", 1024, 67108864);
|
||||
const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900");
|
||||
const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE || "auto");
|
||||
const origin = {
|
||||
@@ -49,7 +66,7 @@ const origin = {
|
||||
browserProxyModeSource: process.env.UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE_SOURCE || "default",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
const webPerformanceRequestBodyMaxBytes = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES, 65536, 1024, 1048576);
|
||||
const webPerformanceRequestBodyMaxBytes = requiredBoundedInteger(process.env.UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES, "UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES", 1024, 1048576);
|
||||
const authLoginMaxAttempts = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_ATTEMPTS, 6, 1, 20);
|
||||
const authLoginRequestTimeoutMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_REQUEST_TIMEOUT_MS, 30000, 1000, 120000);
|
||||
const authLoginInitialDelayMs = boundedInteger(process.env.UNIDESK_WEB_OBSERVE_AUTH_LOGIN_INITIAL_DELAY_MS, 500, 0, 60000);
|
||||
@@ -115,16 +132,25 @@ let browserProcessMonitorSeq = 0;
|
||||
let browserFreezeBlocker = null;
|
||||
let shutdownSignal = null;
|
||||
let browserResourcesClosePromise = null;
|
||||
let browserLaunchAttempted = false;
|
||||
const browserProcessHistory = [];
|
||||
const browserPageRuntimeBaselines = new Map();
|
||||
const browserFreezeSignalHistory = [];
|
||||
let trackedBrowserProcessIdentities = [];
|
||||
const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] };
|
||||
const jsonlWriterState = { queue: [], queuedBytes: 0, inFlight: 0, pump: null, acceptedEntries: 0, acceptedBytes: 0, writtenEntries: 0, writtenBytes: 0, droppedEntries: 0, droppedBytes: 0, failedEntries: 0, dropsByFile: {} };
|
||||
const responseBodyState = { inFlight: 0, accepted: 0, skippedSaturated: 0, completed: 0, failed: 0 };
|
||||
const passiveTaskState = { tasks: new Set(), accepted: 0, dropped: 0, completed: 0, failed: 0, drainTimedOut: false };
|
||||
const runnerSleepWakes = new Set();
|
||||
let lastProcessScan = { ok: false, complete: false, totalPidEntries: 0, scannedPidEntries: 0, failedPidEntries: 0, truncated: false, readConcurrency: maxProcessReadConcurrency, error: "not-scanned", valuesRedacted: true };
|
||||
let realtimeAggregateLast = null;
|
||||
|
||||
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
||||
process.once(signal, () => {
|
||||
shutdownSignal = signal;
|
||||
stopping = true;
|
||||
terminalStatus = "stopping";
|
||||
interruptRunnerSleep();
|
||||
void closeBrowserResources("signal:" + signal);
|
||||
});
|
||||
}
|
||||
@@ -139,6 +165,7 @@ try {
|
||||
if (jsonlRotation.files.length > 0) await appendJsonl(files.control, eventRecord("jsonl-rotated", { stamp: jsonlRotation.stamp, archiveDir: path.relative(stateDir, dirs.archive), files: jsonlRotation.files, valuesRedacted: true }));
|
||||
const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
|
||||
const { chromium } = await launcher.importPlaywright();
|
||||
browserLaunchAttempted = true;
|
||||
browser = await launcher.launchChromium(chromium, chromiumLaunchOptions);
|
||||
context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) });
|
||||
page = await context.newPage();
|
||||
@@ -181,7 +208,7 @@ try {
|
||||
if (browserFreezeBlocker) break;
|
||||
await samplePage("interval");
|
||||
if (browserFreezeBlocker) break;
|
||||
await sleep(sampleIntervalMs);
|
||||
await interruptibleRunnerSleep(sampleIntervalMs);
|
||||
}
|
||||
if (browserFreezeBlocker) {
|
||||
terminalStatus = "failed";
|
||||
@@ -203,7 +230,11 @@ try {
|
||||
} finally {
|
||||
if (browserProcessMonitorStop) browserProcessMonitorStop();
|
||||
if (heartbeatPulseTimer) clearInterval(heartbeatPulseTimer);
|
||||
await closeBrowserResources(shutdownSignal === null ? "finally" : "signal:" + shutdownSignal);
|
||||
interruptRunnerSleep();
|
||||
const closeResult = await closeBrowserResources(shutdownSignal === null ? "finally" : "signal:" + shutdownSignal);
|
||||
const passiveTaskDrain = await drainPassiveTasks();
|
||||
const terminalFlush = await flushJsonlQueue();
|
||||
await writeTerminalResourceStats({ closeResult, passiveTaskDrain, terminalFlush }).catch(() => {});
|
||||
}
|
||||
|
||||
async function closeBrowserResources(reason) {
|
||||
@@ -214,15 +245,32 @@ async function closeBrowserResources(reason) {
|
||||
const closingPage = page;
|
||||
const closingContext = context;
|
||||
const closingBrowser = browser;
|
||||
let preCloseIdentityDiscovery;
|
||||
try {
|
||||
const summary = await collectChromiumProcessSummary(null);
|
||||
preCloseIdentityDiscovery = { ok: true, processScan: summary.processScan, chromiumProcessCount: summary.chromiumProcessCount, valuesRedacted: true };
|
||||
} catch (error) {
|
||||
preCloseIdentityDiscovery = { ok: false, processScan: lastProcessScan, error: errorSummary(error), valuesRedacted: true };
|
||||
}
|
||||
observerPage = null;
|
||||
page = null;
|
||||
context = null;
|
||||
browser = null;
|
||||
if (closingObserverPage && !closingObserverPage.isClosed()) await closingObserverPage.close().catch(() => {});
|
||||
if (closingPage && !closingPage.isClosed()) await closingPage.close().catch(() => {});
|
||||
if (closingContext) await closingContext.close().catch(() => {});
|
||||
if (closingBrowser) await closingBrowser.close().catch(() => {});
|
||||
return { ok: true, reason, valuesRedacted: true };
|
||||
const steps = [];
|
||||
const closeStep = async (label, operation) => {
|
||||
try {
|
||||
await withHardTimeout(Promise.resolve().then(operation), closeStepTimeoutMs, label + " close exceeded " + closeStepTimeoutMs + "ms");
|
||||
steps.push({ label, ok: true, valuesRedacted: true });
|
||||
} catch (error) {
|
||||
steps.push({ label, ok: false, error: errorSummary(error), valuesRedacted: true });
|
||||
}
|
||||
};
|
||||
if (closingObserverPage && !closingObserverPage.isClosed()) await closeStep("observer-page", () => closingObserverPage.close());
|
||||
if (closingPage && !closingPage.isClosed()) await closeStep("control-page", () => closingPage.close());
|
||||
if (closingContext) await closeStep("browser-context", () => closingContext.close());
|
||||
if (closingBrowser) await closeStep("browser", () => closingBrowser.close());
|
||||
const browserCleanup = await terminateTrackedBrowserProcesses("browser-close-residual", preCloseIdentityDiscovery);
|
||||
return { ok: steps.every((step) => step.ok === true) && browserCleanup.ok === true, reason, closeStepTimeoutMs, steps, browserCleanup, valuesRedacted: true };
|
||||
})();
|
||||
return browserResourcesClosePromise;
|
||||
}
|
||||
|
||||
@@ -300,7 +300,7 @@ async function workbenchTraceReadabilityWaitForCurrentProductCard(timeoutMs, pol
|
||||
if (stableCardCount === 1 && timelineCount === 1) return { card, traceId, scope: lastScope };
|
||||
}
|
||||
}
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
const error = new Error("在 owning YAML 指定的 " + timeoutMs + "ms 内未找到当前新运行中的产品 Trace 卡片");
|
||||
error.details = { scope: lastScope, valuesRedacted: true };
|
||||
@@ -313,7 +313,7 @@ async function workbenchTraceReadabilityWaitForDisclosureOpen(card, traceId, tim
|
||||
while (Date.now() <= deadline) {
|
||||
latest = await workbenchTraceReadabilitySnapshot(card, traceId);
|
||||
if (latest.disclosure.open === true) return latest;
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
return latest;
|
||||
}
|
||||
@@ -422,7 +422,7 @@ async function workbenchTraceReadabilityWaitForReadableRunning(card, traceId, ti
|
||||
if (workbenchTraceReadabilityTerminalStatus(latest.status)) {
|
||||
return { observed: false, reason: "terminal-before-readable-running-sample", elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
return { observed: false, reason: "readable-running-timeout", elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
@@ -436,7 +436,7 @@ async function workbenchTraceReadabilityWaitForTerminal(card, traceId, timeoutMs
|
||||
if (workbenchTraceReadabilityTerminalStatus(latest.status)) {
|
||||
return { observed: true, status: latest.status, elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
return { observed: false, status: latest.status, elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
@@ -466,7 +466,7 @@ async function workbenchTraceReadabilityWaitForStableTerminal(card, traceId, qui
|
||||
stableSinceMs = null;
|
||||
}
|
||||
previousSignature = signature;
|
||||
await sleep(pollIntervalMs);
|
||||
await interruptibleRunnerSleep(pollIntervalMs);
|
||||
}
|
||||
return { observed: workbenchTraceReadabilityTerminalStatus(latest.status), stable: false, sampleCount, elapsedMs: Date.now() - startedAtMs, snapshot: latest, valuesRedacted: true };
|
||||
}
|
||||
|
||||
@@ -2,13 +2,221 @@
|
||||
// Responsibility: Runner low-level IO, URL, YAML-derived policy parsing, redaction, and timeout utility source fragment.
|
||||
|
||||
export function nodeWebObserveRunnerUtilitySource(): string {
|
||||
return String.raw`async function appendJsonl(file, value) {
|
||||
await appendFile(file, JSON.stringify(sanitize(value)) + "\n", { mode: 0o600 });
|
||||
return String.raw`function appendJsonl(file, value) {
|
||||
let line;
|
||||
try {
|
||||
line = JSON.stringify(sanitize(value)) + "\n";
|
||||
} catch {
|
||||
recordJsonlDrop(file, 0);
|
||||
return Promise.resolve({ ok: false, dropped: true, reason: "json-encode-failed", valuesRedacted: true });
|
||||
}
|
||||
const bytes = Buffer.byteLength(line);
|
||||
if (bytes > maxJsonlLineBytes) {
|
||||
recordJsonlDrop(file, bytes);
|
||||
return Promise.resolve({ ok: false, dropped: true, reason: "entry-too-large", bytes, maxJsonlLineBytes, valuesRedacted: true });
|
||||
}
|
||||
if (jsonlWriterState.queue.length >= maxJsonlQueueEntries || jsonlWriterState.queuedBytes + bytes > maxJsonlQueueBytes) {
|
||||
recordJsonlDrop(file, bytes);
|
||||
return Promise.resolve({ ok: false, dropped: true, reason: "queue-saturated", bytes, valuesRedacted: true });
|
||||
}
|
||||
jsonlWriterState.acceptedEntries += 1;
|
||||
jsonlWriterState.acceptedBytes += bytes;
|
||||
jsonlWriterState.queuedBytes += bytes;
|
||||
const completion = new Promise((resolve) => jsonlWriterState.queue.push({ file, line, bytes, resolve }));
|
||||
startJsonlPump();
|
||||
return completion;
|
||||
}
|
||||
|
||||
function recordJsonlDrop(file, bytes) {
|
||||
jsonlWriterState.droppedEntries += 1;
|
||||
jsonlWriterState.droppedBytes += Math.max(0, Number(bytes) || 0);
|
||||
const key = path.basename(String(file || "unknown")).slice(0, 80);
|
||||
jsonlWriterState.dropsByFile[key] = (jsonlWriterState.dropsByFile[key] || 0) + 1;
|
||||
}
|
||||
|
||||
function startJsonlPump() {
|
||||
if (jsonlWriterState.pump) return jsonlWriterState.pump;
|
||||
jsonlWriterState.pump = (async () => {
|
||||
while (jsonlWriterState.queue.length > 0) {
|
||||
const entry = jsonlWriterState.queue.shift();
|
||||
jsonlWriterState.queuedBytes = Math.max(0, jsonlWriterState.queuedBytes - entry.bytes);
|
||||
jsonlWriterState.inFlight = 1;
|
||||
try {
|
||||
await appendFile(entry.file, entry.line, { mode: 0o600 });
|
||||
jsonlWriterState.writtenEntries += 1;
|
||||
jsonlWriterState.writtenBytes += entry.bytes;
|
||||
entry.resolve({ ok: true, dropped: false, valuesRedacted: true });
|
||||
} catch (error) {
|
||||
jsonlWriterState.failedEntries += 1;
|
||||
entry.resolve({ ok: false, dropped: false, error: errorSummary(error), valuesRedacted: true });
|
||||
} finally {
|
||||
jsonlWriterState.inFlight = 0;
|
||||
}
|
||||
}
|
||||
})().finally(() => {
|
||||
jsonlWriterState.pump = null;
|
||||
if (jsonlWriterState.queue.length > 0) startJsonlPump();
|
||||
});
|
||||
return jsonlWriterState.pump;
|
||||
}
|
||||
|
||||
function jsonlWriterStats() {
|
||||
return {
|
||||
queueEntries: jsonlWriterState.queue.length,
|
||||
queueBytes: jsonlWriterState.queuedBytes,
|
||||
inFlight: jsonlWriterState.inFlight,
|
||||
maxQueueEntries: maxJsonlQueueEntries,
|
||||
maxQueueBytes: maxJsonlQueueBytes,
|
||||
maxLineBytes: maxJsonlLineBytes,
|
||||
acceptedEntries: jsonlWriterState.acceptedEntries,
|
||||
acceptedBytes: jsonlWriterState.acceptedBytes,
|
||||
writtenEntries: jsonlWriterState.writtenEntries,
|
||||
writtenBytes: jsonlWriterState.writtenBytes,
|
||||
droppedEntries: jsonlWriterState.droppedEntries,
|
||||
droppedBytes: jsonlWriterState.droppedBytes,
|
||||
failedEntries: jsonlWriterState.failedEntries,
|
||||
dropsByFile: { ...jsonlWriterState.dropsByFile },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function flushJsonlQueue() {
|
||||
try {
|
||||
await withHardTimeout((async () => {
|
||||
while (jsonlWriterState.queue.length > 0 || jsonlWriterState.inFlight > 0 || jsonlWriterState.pump) {
|
||||
if (jsonlWriterState.pump) await jsonlWriterState.pump;
|
||||
else if (jsonlWriterState.queue.length > 0) await startJsonlPump();
|
||||
else break;
|
||||
}
|
||||
})(), jsonlFlushTimeoutMs, "jsonl terminal flush exceeded " + jsonlFlushTimeoutMs + "ms");
|
||||
return { ok: true, timedOut: false, stats: jsonlWriterStats(), valuesRedacted: true };
|
||||
} catch (error) {
|
||||
return { ok: false, timedOut: true, error: errorSummary(error), stats: jsonlWriterStats(), valuesRedacted: true };
|
||||
}
|
||||
}
|
||||
|
||||
async function fileMeta(file) {
|
||||
const [buffer, stats] = await Promise.all([readFile(file), stat(file)]);
|
||||
return { byteCount: stats.size, sha256: "sha256:" + createHash("sha256").update(buffer).digest("hex") };
|
||||
const digestPromise = new Promise((resolve, reject) => {
|
||||
const digest = createHash("sha256");
|
||||
const stream = createReadStream(file);
|
||||
stream.on("data", (chunk) => digest.update(chunk));
|
||||
stream.once("error", reject);
|
||||
stream.once("end", () => resolve("sha256:" + digest.digest("hex")));
|
||||
});
|
||||
const [sha256, stats] = await Promise.all([digestPromise, stat(file)]);
|
||||
return { byteCount: stats.size, sha256 };
|
||||
}
|
||||
|
||||
function responseBodyReadContract(response, maxBytes = responseBodyMaxBytes) {
|
||||
let headers = {};
|
||||
try { headers = response?.headers?.() || {}; } catch {}
|
||||
const contentEncoding = String(headers["content-encoding"] || headers["Content-Encoding"] || "").trim().toLowerCase();
|
||||
if (contentEncoding && contentEncoding !== "identity") {
|
||||
return { ok: false, reason: "encoded-body-size-unbounded", contentEncoding: truncate(contentEncoding, 80), contentLength: null, maxBytes, valuesRedacted: true };
|
||||
}
|
||||
const rawContentLength = headers["content-length"] || headers["Content-Length"];
|
||||
const contentLength = Number(rawContentLength);
|
||||
if (!Number.isFinite(contentLength) || contentLength < 0) {
|
||||
return { ok: false, reason: "content-length-unavailable", contentLength: null, maxBytes, valuesRedacted: true };
|
||||
}
|
||||
if (contentLength > maxBytes) {
|
||||
return { ok: false, reason: "content-length-too-large", contentLength, maxBytes, valuesRedacted: true };
|
||||
}
|
||||
return { ok: true, reason: null, contentLength, maxBytes, valuesRedacted: true };
|
||||
}
|
||||
|
||||
async function boundedResponseJson(response, maxBytes = responseBodyMaxBytes) {
|
||||
const bodyContract = responseBodyReadContract(response, maxBytes);
|
||||
if (bodyContract.ok !== true) return { ok: false, value: null, error: bodyContract, bodyContract, valuesRedacted: true };
|
||||
const permit = tryAcquireResponseBodyPermit();
|
||||
if (!permit) return { ok: false, value: null, error: { reason: "response-body-inflight-saturated", valuesRedacted: true }, bodyContract, valuesRedacted: true };
|
||||
try {
|
||||
const value = await response.json();
|
||||
responseBodyState.completed += 1;
|
||||
return { ok: true, value, error: null, bodyContract, valuesRedacted: true };
|
||||
} catch (error) {
|
||||
responseBodyState.failed += 1;
|
||||
return { ok: false, value: null, error: errorSummary(error), bodyContract, valuesRedacted: true };
|
||||
} finally {
|
||||
permit.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function boundedResponseText(response, maxBytes = responseBodyMaxBytes) {
|
||||
const bodyContract = responseBodyReadContract(response, maxBytes);
|
||||
if (bodyContract.ok !== true) return { ok: false, value: null, error: bodyContract, bodyContract, valuesRedacted: true };
|
||||
const permit = tryAcquireResponseBodyPermit();
|
||||
if (!permit) return { ok: false, value: null, error: { reason: "response-body-inflight-saturated", valuesRedacted: true }, bodyContract, valuesRedacted: true };
|
||||
try {
|
||||
const value = await response.text();
|
||||
responseBodyState.completed += 1;
|
||||
return { ok: true, value, error: null, bodyContract, valuesRedacted: true };
|
||||
} catch (error) {
|
||||
responseBodyState.failed += 1;
|
||||
return { ok: false, value: null, error: errorSummary(error), bodyContract, valuesRedacted: true };
|
||||
} finally {
|
||||
permit.release();
|
||||
}
|
||||
}
|
||||
|
||||
function tryAcquireResponseBodyPermit() {
|
||||
if (responseBodyState.inFlight >= maxResponseBodyInFlight) {
|
||||
responseBodyState.skippedSaturated += 1;
|
||||
return null;
|
||||
}
|
||||
responseBodyState.inFlight += 1;
|
||||
responseBodyState.accepted += 1;
|
||||
let released = false;
|
||||
return { release() { if (released) return; released = true; responseBodyState.inFlight = Math.max(0, responseBodyState.inFlight - 1); } };
|
||||
}
|
||||
|
||||
function responseBodyStats() {
|
||||
return { ...responseBodyState, maxInFlight: maxResponseBodyInFlight, valuesRedacted: true };
|
||||
}
|
||||
|
||||
function trackPassiveTask(factory, label) {
|
||||
if (stopping || passiveTaskState.tasks.size >= maxPassiveListenerTasks) {
|
||||
passiveTaskState.dropped += 1;
|
||||
return false;
|
||||
}
|
||||
passiveTaskState.accepted += 1;
|
||||
let task;
|
||||
task = Promise.resolve()
|
||||
.then(factory)
|
||||
.then(() => { passiveTaskState.completed += 1; })
|
||||
.catch(async (error) => {
|
||||
passiveTaskState.failed += 1;
|
||||
await appendJsonl(files.errors, eventRecord("passive-task-error", { label, error: errorSummary(error), valuesRedacted: true })).catch(() => {});
|
||||
})
|
||||
.finally(() => passiveTaskState.tasks.delete(task));
|
||||
passiveTaskState.tasks.add(task);
|
||||
task.catch(() => {});
|
||||
return true;
|
||||
}
|
||||
|
||||
function passiveTaskStats() {
|
||||
return {
|
||||
inFlight: passiveTaskState.tasks.size,
|
||||
maxInFlight: maxPassiveListenerTasks,
|
||||
accepted: passiveTaskState.accepted,
|
||||
dropped: passiveTaskState.dropped,
|
||||
completed: passiveTaskState.completed,
|
||||
failed: passiveTaskState.failed,
|
||||
drainTimedOut: passiveTaskState.drainTimedOut,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function drainPassiveTasks() {
|
||||
try {
|
||||
await withHardTimeout((async () => {
|
||||
while (passiveTaskState.tasks.size > 0) await Promise.allSettled([...passiveTaskState.tasks]);
|
||||
})(), passiveTaskFlushTimeoutMs, "passive task drain exceeded " + passiveTaskFlushTimeoutMs + "ms");
|
||||
return { ok: true, timedOut: false, stats: passiveTaskStats(), valuesRedacted: true };
|
||||
} catch (error) {
|
||||
passiveTaskState.drainTimedOut = true;
|
||||
return { ok: false, timedOut: true, error: errorSummary(error), stats: passiveTaskStats(), valuesRedacted: true };
|
||||
}
|
||||
}
|
||||
|
||||
function currentPageUrl() {
|
||||
@@ -77,6 +285,14 @@ function boundedInteger(value, fallback, min, max) {
|
||||
return integer;
|
||||
}
|
||||
|
||||
function requiredBoundedInteger(value, name, min, max) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed < min || parsed > max) {
|
||||
throw new Error(name + " must come from the owning YAML and be an integer between " + min + " and " + max);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function positiveNumber(value, fallback) {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
@@ -306,9 +522,33 @@ function errorSummary(error) {
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return interruptibleRunnerSleep(ms);
|
||||
}
|
||||
|
||||
function boundedCleanupSleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
|
||||
}
|
||||
|
||||
function interruptibleRunnerSleep(ms) {
|
||||
if (stopping) return Promise.resolve();
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
runnerSleepWakes.delete(wake);
|
||||
resolve();
|
||||
}, Math.max(0, ms));
|
||||
const wake = () => {
|
||||
clearTimeout(timer);
|
||||
runnerSleepWakes.delete(wake);
|
||||
resolve();
|
||||
};
|
||||
runnerSleepWakes.add(wake);
|
||||
});
|
||||
}
|
||||
|
||||
function interruptRunnerSleep() {
|
||||
for (const wake of [...runnerSleepWakes]) wake();
|
||||
}
|
||||
|
||||
function withHardTimeout(promise, timeoutMs, message) {
|
||||
const guarded = Promise.resolve(promise);
|
||||
let timer = null;
|
||||
|
||||
@@ -1285,12 +1285,10 @@ async function launchWorkbenchFromTask(command) {
|
||||
}, { timeout: 30000 }).then(async (response) => {
|
||||
let chatPayload = null;
|
||||
let chatPayloadError = null;
|
||||
try {
|
||||
chatPayload = await response.json();
|
||||
} catch (error) {
|
||||
chatPayloadError = errorSummary(error);
|
||||
}
|
||||
const headers = response.headers();
|
||||
const chatBody = await boundedResponseJson(response);
|
||||
chatPayload = chatBody.value;
|
||||
chatPayloadError = chatBody.error;
|
||||
return {
|
||||
observed: true,
|
||||
status: response.status(),
|
||||
@@ -1316,11 +1314,9 @@ async function launchWorkbenchFromTask(command) {
|
||||
const launchTraceHeader = typeof headers["x-hwlab-otel-trace-id"] === "string" ? headers["x-hwlab-otel-trace-id"] : null;
|
||||
let payload = null;
|
||||
let responseParseError = null;
|
||||
try {
|
||||
payload = await launchResponse.json();
|
||||
} catch (error) {
|
||||
responseParseError = errorSummary(error);
|
||||
}
|
||||
const launchBody = await boundedResponseJson(launchResponse);
|
||||
payload = launchBody.value;
|
||||
responseParseError = launchBody.error;
|
||||
const sessionId = sessionIdFromAgentSessionPayload(payload);
|
||||
const workbenchUrl = safeUrlPath(payload?.workbenchUrl || payload?.url || "");
|
||||
const contractVersion = typeof payload?.contractVersion === "string" ? payload.contractVersion : null;
|
||||
|
||||
@@ -48,10 +48,13 @@ test("manual start is blocked at exactly 4 GiB and requires controlled GC then M
|
||||
assert.equal(memory.comparator, "less-than-or-equal");
|
||||
assert.equal(memory.thresholdMatched, true);
|
||||
assert.equal(gcNext.requiredBeforeStart, true);
|
||||
assert.match(String(gcNext.planCommand), /gc remote NC01 plan$/u);
|
||||
assert.match(String(gcNext.runCommand), /gc remote NC01 run --confirm$/u);
|
||||
assert.match(String(gcNext.planCommand), /gc remote NC01 plan --memory-pressure-only$/u);
|
||||
assert.match(String(gcNext.runCommand), /gc remote NC01 run --confirm --memory-pressure-only$/u);
|
||||
assert.match(String(gcNext.statusCommandTemplate), /gc remote NC01 status --job-id <job-id>$/u);
|
||||
assert.match(String(gcNext.recheckCommand), /gc remote NC01 memory$/u);
|
||||
assert.equal(gcNext.recheckMetric, "MemAvailable");
|
||||
assert.deepEqual(gcNext.requiredOrder, ["plan", "run", "status-until-terminal", "memory-recheck"]);
|
||||
assert.deepEqual(gcNext.startRemainsBlockedUntil, ["gc-job-terminal", "MemAvailable-threshold-cleared"]);
|
||||
});
|
||||
|
||||
test("sentinel allows exactly 4 GiB but skips one byte below without considering swap", () => {
|
||||
@@ -127,7 +130,7 @@ test("manual and sentinel gates return before any observer launch path", () => {
|
||||
const sentinelSource = readFileSync(rootPath("scripts/src/hwlab-node-web-sentinel-p5-observe.ts"), "utf8");
|
||||
const quickVerify = sentinelSource.indexOf("export function runSentinelQuickVerify");
|
||||
const sentinelGuard = sentinelSource.indexOf("runWebProbeMemoryGuard(state.spec, \"sentinel-cadence\")", quickVerify);
|
||||
const sentinelSkip = sentinelSource.indexOf('status: "skipped-wait-next-round"', sentinelGuard);
|
||||
const sentinelSkip = sentinelSource.indexOf("return recordSkippedQuickVerify", sentinelGuard);
|
||||
const providerPreflight = sentinelSource.indexOf("runSentinelExactProviderProfilePreflight", quickVerify);
|
||||
const observeStart = sentinelSource.indexOf('"web-probe", "observe", "start"', quickVerify);
|
||||
assert.ok(quickVerify >= 0 && sentinelGuard > quickVerify && sentinelSkip > sentinelGuard);
|
||||
|
||||
@@ -39,13 +39,19 @@ function humanBytes(value: number | null): string | null {
|
||||
function controlledGcNext(node: string): Record<string, unknown> {
|
||||
return {
|
||||
automaticGc: false,
|
||||
requiredOrder: ["plan", "run", "recheck"],
|
||||
planCommand: `bun scripts/cli.ts gc remote ${node} plan`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${node} run --confirm`,
|
||||
requiredOrder: ["plan", "run", "status-until-terminal", "memory-recheck"],
|
||||
scope: "memory-pressure-only",
|
||||
planCommand: `bun scripts/cli.ts gc remote ${node} plan --memory-pressure-only`,
|
||||
runCommand: `bun scripts/cli.ts gc remote ${node} run --confirm --memory-pressure-only`,
|
||||
jobIdSource: "gc remote run result.data.jobId",
|
||||
statusCommandTemplate: `bun scripts/cli.ts gc remote ${node} status --job-id <job-id>`,
|
||||
terminalStatuses: ["succeeded", "failed", "completed", "blocked"],
|
||||
statusRequiredWhenJobIdReturned: true,
|
||||
recheckCommand: `bun scripts/cli.ts gc remote ${node} memory`,
|
||||
recheckMetric: "MemAvailable",
|
||||
recheckSource: "/proc/meminfo",
|
||||
swapExcluded: true,
|
||||
startRemainsBlockedUntil: ["gc-job-terminal", "MemAvailable-threshold-cleared"],
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -439,6 +439,7 @@ function knownWebProbeFindingIds(): string[] {
|
||||
"scripts/src/hwlab-node-web-observe-analyzer-window-page-source.ts",
|
||||
"scripts/src/hwlab-node-web-observe-analyzer-workbench-triad-source.ts",
|
||||
"scripts/src/hwlab-node-web-sentinel-p5-observe.ts",
|
||||
"scripts/src/hwlab-node-web-sentinel-p5-observe-analysis.ts",
|
||||
]) {
|
||||
let source = "";
|
||||
try {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,7 @@ import {
|
||||
quickVerifyCleanupFindings,
|
||||
quickVerifyUnhandledExceptionSummary,
|
||||
readAnalysisSummaryFromWorkspace,
|
||||
sentinelChildStartMemorySkip,
|
||||
sentinelServiceProxyInvocation,
|
||||
stopQuickVerifyObserverAfterPartialStart,
|
||||
} from "./hwlab-node-web-sentinel-p5-observe";
|
||||
@@ -69,6 +70,7 @@ test("recorded sentinel run is available through the latest report index", () =>
|
||||
views: { summary: { renderedText: "indexed summary" } },
|
||||
findingCount: 0,
|
||||
artifactCount: 0,
|
||||
updatedAt: "2026-07-13T00:00:00.000Z",
|
||||
valuesRedacted: true,
|
||||
});
|
||||
const latest = service.report("summary", null);
|
||||
@@ -100,6 +102,7 @@ test("latest report selects the newest stored run without a report SHA", () => {
|
||||
views: { summary: { renderedText: "old indexed summary" } },
|
||||
findingCount: 0,
|
||||
artifactCount: 0,
|
||||
updatedAt: "2026-07-13T00:00:00.000Z",
|
||||
valuesRedacted: true,
|
||||
});
|
||||
service.recordRun({
|
||||
@@ -111,6 +114,7 @@ test("latest report selects the newest stored run without a report SHA", () => {
|
||||
findings: [{ id: "quick-verify-observer-start-failed", severity: "error", summary: "observer start failed" }],
|
||||
findingCount: 1,
|
||||
artifactCount: 0,
|
||||
updatedAt: "2026-07-13T00:00:01.000Z",
|
||||
valuesRedacted: true,
|
||||
});
|
||||
|
||||
@@ -255,7 +259,7 @@ test("quick verify observer lifecycle stops once on success, failure, and except
|
||||
}
|
||||
});
|
||||
|
||||
test("quick verify observer lifecycle records a thrown stop without retrying it", () => {
|
||||
test("quick verify observer lifecycle retries a failed stop again from the bounded finally fallback", () => {
|
||||
let calls = 0;
|
||||
const lifecycle = createQuickVerifyObserverLifecycle("webobs-stop-throws", () => {
|
||||
calls += 1;
|
||||
@@ -265,12 +269,28 @@ test("quick verify observer lifecycle records a thrown stop without retrying it"
|
||||
const first = lifecycle.stop("observe-stop-after-failure");
|
||||
const fallback = lifecycle.stop("observe-stop-after-finally");
|
||||
|
||||
assert.equal(calls, 1);
|
||||
assert.strictEqual(first, fallback);
|
||||
assert.equal(calls, 4);
|
||||
assert.notStrictEqual(first, fallback);
|
||||
assert.equal(first.failure, "observer-stop-unhandled-exception");
|
||||
assert.equal(Array.isArray(fallback.stopAttempts), true);
|
||||
assert.equal((fallback.stopAttempts as unknown[]).length, 4);
|
||||
assert.equal(JSON.stringify(first).includes("sensitive-stop-error"), false);
|
||||
});
|
||||
|
||||
test("quick verify observer lifecycle retries a failed force-stop and caches the successful fallback", () => {
|
||||
let calls = 0;
|
||||
const lifecycle = createQuickVerifyObserverLifecycle("webobs-stop-retry", (_observerId, phase) => {
|
||||
calls += 1;
|
||||
return { phase, ok: calls === 2, observerId: "webobs-stop-retry" };
|
||||
});
|
||||
const first = lifecycle.stop("observe-stop-after-failure");
|
||||
const cached = lifecycle.stop("observe-stop-after-finally");
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(first.ok, true);
|
||||
assert.strictEqual(first, cached);
|
||||
assert.equal(Array.isArray(first.stopAttempts), true);
|
||||
});
|
||||
|
||||
test("quick verify cleans a partial start only when start output contains an observer id", () => {
|
||||
const calls: { observerId: string; phase: string }[] = [];
|
||||
const stopObserver = (observerId: string, phase: string) => {
|
||||
@@ -296,19 +316,66 @@ test("quick verify unexpected exception evidence is bounded and redacted", () =>
|
||||
assert.equal(serialized.includes(secret), false);
|
||||
});
|
||||
|
||||
test("sentinel preserves a child launch-time memory race as skipped instead of observe-start-failed", () => {
|
||||
const skip = sentinelChildStartMemorySkip({
|
||||
ok: true,
|
||||
command: "web-probe observe start",
|
||||
data: {
|
||||
ok: true,
|
||||
status: "skipped-wait-next-round",
|
||||
observerCreated: false,
|
||||
memoryGuard: {
|
||||
mode: "sentinel-cadence",
|
||||
status: "skipped",
|
||||
memory: {
|
||||
metric: "MemAvailable",
|
||||
source: "/proc/meminfo",
|
||||
swapExcluded: true,
|
||||
availableBytes: 4 * 1024 ** 3 - 1,
|
||||
thresholdBytes: 4 * 1024 ** 3,
|
||||
comparator: "less-than",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(skip?.status, "skipped-wait-next-round");
|
||||
assert.equal(skip?.decision, "skip-observer");
|
||||
assert.equal(skip?.observerCreated, false);
|
||||
assert.equal((skip?.memoryGuard as Record<string, unknown>).mode, "sentinel-cadence");
|
||||
assert.equal(sentinelChildStartMemorySkip({ data: { status: "blocked", observerCreated: false } }), null);
|
||||
});
|
||||
|
||||
test("web observe runner records finite lifecycle policy and closes the browser", () => {
|
||||
const source = nodeWebObserveRunnerSource();
|
||||
|
||||
assert.match(source, /sampling:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
|
||||
assert.match(source, /observerLifecycle:\s*\{[^}]*maxSamples,[^}]*maxRunMs,[^}]*maxRunSeconds:/u);
|
||||
assert.match(source, /if \(maxRunMs > 0 && Date\.now\(\) - startedAtMs >= maxRunMs\)/u);
|
||||
assert.match(source, /const maxScreenshots = boundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS/u);
|
||||
assert.match(source, /const maxPerformanceEntries = boundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES/u);
|
||||
assert.match(source, /const maxScreenshots = requiredBoundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_SCREENSHOTS/u);
|
||||
assert.match(source, /const maxPerformanceEntries = requiredBoundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES/u);
|
||||
assert.match(source, /const maxRealtimeEventBodyBytes = requiredBoundedInteger\(process\.env\.UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES/u);
|
||||
assert.match(source, /while \(browserProcessHistory\.length > maxBrowserProcessHistoryEntries\)/u);
|
||||
assert.match(source, /while \(browserFreezeSignalHistory\.length > maxBrowserFreezeSignalHistoryEntries\)/u);
|
||||
assert.match(source, /if \(screenshotCount >= maxScreenshots\)/u);
|
||||
assert.match(source, /const maxBytes = responseBodyMaxBytes/u);
|
||||
assert.match(source, /function responseBodyReadContract\(response, maxBytes = responseBodyMaxBytes\)/u);
|
||||
assert.match(source, /contentEncoding[\s\S]*?reason: "encoded-body-size-unbounded"/u);
|
||||
assert.match(source, /!Number\.isFinite\(contentLength\)[\s\S]*?reason: "content-length-unavailable"/u);
|
||||
assert.match(source, /bodyReadSkipped: body\.error\?\.reason \|\| "bounded-response-read-failed"/u);
|
||||
assert.match(source, /\.slice\(0, maxProcessScanEntries\)/u);
|
||||
assert.match(source, /if \(items\.length > maxEntries\)[\s\S]*?items\.splice/u);
|
||||
assert.match(source, /process\.once\(signal,[\s\S]*?closeBrowserResources\("signal:" \+ signal\)/u);
|
||||
assert.match(source, /finally \{[\s\S]*?await closeBrowserResources/u);
|
||||
assert.match(source, /closingObserverPage\.close\(\)[\s\S]*?closingPage\.close\(\)[\s\S]*?closingContext\.close\(\)[\s\S]*?closingBrowser\.close\(\)/u);
|
||||
|
||||
const directory = mkdtempSync(join(tmpdir(), "web-observe-runner-check-"));
|
||||
try {
|
||||
const runner = join(directory, "observer-runner.mjs");
|
||||
writeFileSync(runner, source);
|
||||
const syntax = spawnSync("node", ["--check", runner], { encoding: "utf8" });
|
||||
assert.equal(syntax.status, 0, syntax.stderr);
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2728,7 +2728,7 @@ function reportRunView(config: WebProbeSentinelServiceConfig, db: Database, view
|
||||
return { ok: false, error: "unsupported-report-view", view, valuesRedacted: true };
|
||||
}
|
||||
const row = runId === null
|
||||
? db.query("SELECT * FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY updated_at DESC LIMIT 1").get(config.sentinelId, config.node, config.lane) as Record<string, unknown> | null
|
||||
? db.query("SELECT * FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY updated_at DESC, rowid DESC LIMIT 1").get(config.sentinelId, config.node, config.lane) as Record<string, unknown> | null
|
||||
: readRunRow(config, db, runId);
|
||||
if (row === null) return { ok: false, error: "report-run-missing", runId, view, valuesRedacted: true };
|
||||
const selectedRunId = stringOrNull(row.id);
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { resolveCliChildJsonCommandResult, type CliChildJsonResolution } from "../cli-child-json-recovery";
|
||||
import type { HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
||||
import { runWebProbeMemoryGuard } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { evaluateWebProbeMemoryGuard, runWebProbeMemoryGuard } from "../hwlab-node-web-probe-memory-guard";
|
||||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||||
import { withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
||||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
||||
@@ -455,6 +455,134 @@ console.log(JSON.stringify({
|
||||
`;
|
||||
}
|
||||
|
||||
export function nodeWebObserveFinalLaunchGuardSource(): string {
|
||||
return String.raw`
|
||||
const fs = require("fs");
|
||||
const thresholdBytes = Number(process.argv[2]);
|
||||
const comparator = process.argv[3];
|
||||
const mode = process.argv[4];
|
||||
const jobId = process.argv[5];
|
||||
const stateDir = process.argv[6];
|
||||
const source = "/proc/meminfo";
|
||||
|
||||
function finish(reason, availableBytes) {
|
||||
const valid = Number.isSafeInteger(availableBytes) && availableBytes >= 0;
|
||||
const thresholdMatched = valid
|
||||
? comparator === "less-than" ? availableBytes < thresholdBytes : availableBytes <= thresholdBytes
|
||||
: null;
|
||||
const blocked = reason !== null || thresholdMatched === true;
|
||||
const cadenceSkip = blocked && mode === "sentinel-cadence";
|
||||
const payload = {
|
||||
ok: !blocked || cadenceSkip,
|
||||
command: "web-probe-observe start",
|
||||
status: blocked ? cadenceSkip ? "skipped-wait-next-round" : "blocked" : "allowed",
|
||||
reason: reason || (thresholdMatched ? "memavailable-threshold-matched" : null),
|
||||
failureKind: blocked ? "web-probe-final-memory-start-guard" : null,
|
||||
jobId,
|
||||
stateDir,
|
||||
observerCreated: false,
|
||||
browserCreated: false,
|
||||
startBlocked: blocked,
|
||||
decision: blocked ? cadenceSkip ? "skip-observer" : "gc-required" : "start-observer",
|
||||
mutation: blocked,
|
||||
stateArtifactCreated: blocked,
|
||||
memoryGuard: {
|
||||
phase: "remote-launch-critical-section",
|
||||
metric: "MemAvailable",
|
||||
source,
|
||||
swapExcluded: true,
|
||||
availableBytes: valid ? availableBytes : null,
|
||||
thresholdBytes,
|
||||
comparator,
|
||||
thresholdMatched,
|
||||
},
|
||||
observedAt: new Date().toISOString(),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
console.log(JSON.stringify(payload));
|
||||
process.exit(blocked ? 42 : 0);
|
||||
}
|
||||
|
||||
if (!Number.isSafeInteger(thresholdBytes) || thresholdBytes <= 0) finish("memory-threshold-invalid", null);
|
||||
if (comparator !== "less-than" && comparator !== "less-than-or-equal") finish("memory-comparator-invalid", null);
|
||||
let text;
|
||||
try { text = fs.readFileSync(source, "utf8"); } catch { finish("proc-meminfo-read-failed", null); }
|
||||
const match = text.match(/^MemAvailable:\s+(\d+)\s+kB$/m);
|
||||
if (!match) finish("memavailable-missing", null);
|
||||
const availableBytes = Number(match[1]) * 1024;
|
||||
finish(null, availableBytes);
|
||||
`;
|
||||
}
|
||||
|
||||
export function nodeWebObserveLaunchReadinessSource(): string {
|
||||
return String.raw`
|
||||
const fs = require("fs");
|
||||
const runnerPid = Number(process.argv[2]);
|
||||
const stateDir = process.argv[3];
|
||||
const expectedJobId = process.argv[4];
|
||||
const timeoutMs = Number(process.argv[5]);
|
||||
const pollMs = Number(process.argv[6]);
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
const readJson = (name) => { try { return JSON.parse(fs.readFileSync(stateDir + "/" + name, "utf8")); } catch { return null; } };
|
||||
|
||||
function procRows() {
|
||||
const rows = new Map();
|
||||
let names = [];
|
||||
try { names = fs.readdirSync("/proc").filter((name) => /^\d+$/.test(name)); } catch { return rows; }
|
||||
for (const name of names) {
|
||||
try {
|
||||
const stat = fs.readFileSync("/proc/" + name + "/stat", "utf8");
|
||||
const close = stat.lastIndexOf(")");
|
||||
const fields = stat.slice(close + 2).trim().split(/\s+/);
|
||||
const command = fs.readFileSync("/proc/" + name + "/cmdline").toString("utf8").replace(/\0/g, " ");
|
||||
rows.set(Number(name), { pid: Number(name), ppid: Number(fields[1]), command });
|
||||
} catch {}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function descendants(rows) {
|
||||
const selected = new Set([runnerPid]);
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const row of rows.values()) if (!selected.has(row.pid) && selected.has(row.ppid)) { selected.add(row.pid); changed = true; }
|
||||
}
|
||||
return [...selected].map((pid) => rows.get(pid)).filter(Boolean);
|
||||
}
|
||||
|
||||
function matchingStateRecord(name, value) {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const status = String(value.status || "");
|
||||
if (!/^(starting|running|ready)$/i.test(status)) return null;
|
||||
if (String(value.jobId || "") !== expectedJobId || Number(value.pid) !== runnerPid) return null;
|
||||
return { name, status, jobId: String(value.jobId), pid: Number(value.pid) };
|
||||
}
|
||||
|
||||
while (Date.now() <= deadline) {
|
||||
const rows = procRows();
|
||||
const runnerAlive = rows.has(runnerPid);
|
||||
const children = runnerAlive ? descendants(rows) : [];
|
||||
const browserChild = children.find((row) => row.pid !== runnerPid && /(?:^|[ /])(chrome|chromium)(?:[ /]|$)/i.test(row.command));
|
||||
const heartbeat = readJson("heartbeat.json");
|
||||
const manifest = readJson("manifest.json");
|
||||
const stateRecord = matchingStateRecord("heartbeat.json", heartbeat) || matchingStateRecord("manifest.json", manifest);
|
||||
if (runnerAlive && browserChild && stateRecord) {
|
||||
console.log(JSON.stringify({ ok: true, expectedJobId, runnerPid, runnerAlive: true, browserPid: browserChild.pid, stateRecord, readiness: "current-state-and-browser-descendant", observedAt: new Date().toISOString(), valuesRedacted: true }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (!runnerAlive) {
|
||||
console.log(JSON.stringify({ ok: false, reason: "runner-exited-before-ready", expectedJobId, runnerPid, runnerAlive: false, stateRecord, observedAt: new Date().toISOString(), valuesRedacted: true }));
|
||||
process.exit(43);
|
||||
}
|
||||
pause(pollMs);
|
||||
}
|
||||
console.log(JSON.stringify({ ok: false, reason: "launch-readiness-timeout", expectedJobId, runnerPid, runnerAlive: fs.existsSync("/proc/" + runnerPid), observedAt: new Date().toISOString(), valuesRedacted: true }));
|
||||
process.exit(44);
|
||||
`;
|
||||
}
|
||||
|
||||
export function runNodeWebProbeObserveStart(
|
||||
options: NodeWebProbeObserveOptions,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
@@ -482,7 +610,9 @@ export function runNodeWebProbeObserveStart(
|
||||
};
|
||||
}
|
||||
const observerLifecycle = spec.webProbe?.resourcePolicy?.observerLifecycle;
|
||||
const finalMemoryPolicy = spec.webProbe?.resourcePolicy?.memoryStartGuard;
|
||||
if (observerLifecycle === undefined) throw new Error("web-probe observer lifecycle policy is missing from the owning YAML");
|
||||
if (finalMemoryPolicy === undefined) throw new Error("web-probe memory start policy is missing from the owning YAML");
|
||||
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
||||
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
|
||||
const day = timestamp.slice(0, 8);
|
||||
@@ -520,8 +650,24 @@ export function runNodeWebProbeObserveStart(
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_ENTRIES=${shellQuote(String(observerLifecycle.maxPerformanceEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_PROCESS_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserProcessHistoryEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_BROWSER_FREEZE_SIGNAL_HISTORY_ENTRIES=${shellQuote(String(observerLifecycle.maxBrowserFreezeSignalHistoryEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_ENTRIES=${shellQuote(String(observerLifecycle.maxRealtimeEventEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_EVENT_BODY_BYTES=${shellQuote(String(observerLifecycle.maxRealtimeEventBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_DOM_EVIDENCE_ROWS=${shellQuote(String(observerLifecycle.maxDomEvidenceRows))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PROCESS_SCAN_ENTRIES=${shellQuote(String(observerLifecycle.maxProcessScanEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PERFORMANCE_CAPTURE_SECONDS=${shellQuote(String(observerLifecycle.maxPerformanceCaptureSeconds))}`,
|
||||
`UNIDESK_WEB_OBSERVE_RESPONSE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxResponseBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_WEB_PERFORMANCE_BODY_MAX_BYTES=${shellQuote(String(observerLifecycle.maxWebPerformanceBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_ENTRIES=${shellQuote(String(observerLifecycle.maxJsonlQueueEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_JSONL_QUEUE_BYTES=${shellQuote(String(observerLifecycle.maxJsonlQueueBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_JSONL_LINE_BYTES=${shellQuote(String(observerLifecycle.maxJsonlLineBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_JSONL_FLUSH_TIMEOUT_MS=${shellQuote(String(observerLifecycle.jsonlFlushTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_RESPONSE_BODY_IN_FLIGHT=${shellQuote(String(observerLifecycle.maxResponseBodyInFlight))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PASSIVE_LISTENER_TASKS=${shellQuote(String(observerLifecycle.maxPassiveListenerTasks))}`,
|
||||
`UNIDESK_WEB_OBSERVE_PASSIVE_TASK_FLUSH_TIMEOUT_MS=${shellQuote(String(observerLifecycle.passiveTaskFlushTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_CLOSE_STEP_TIMEOUT_MS=${shellQuote(String(observerLifecycle.closeStepTimeoutMs))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_PROCESS_READ_CONCURRENCY=${shellQuote(String(observerLifecycle.maxProcessReadConcurrency))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_ENTRIES=${shellQuote(String(observerLifecycle.maxRealtimeTotalEntries))}`,
|
||||
`UNIDESK_WEB_OBSERVE_MAX_REALTIME_TOTAL_BODY_BYTES=${shellQuote(String(observerLifecycle.maxRealtimeTotalBodyBytes))}`,
|
||||
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
|
||||
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
|
||||
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
|
||||
@@ -539,6 +685,26 @@ export function runNodeWebProbeObserveStart(
|
||||
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`,
|
||||
]),
|
||||
].join(" ");
|
||||
const finalComparator = options.memoryGuardMode === "sentinel-cadence"
|
||||
? finalMemoryPolicy.sentinelComparator
|
||||
: finalMemoryPolicy.manualComparator;
|
||||
const finalGuardFailure = (reason: string) => JSON.stringify({
|
||||
ok: false,
|
||||
command: "web-probe-observe start",
|
||||
status: "blocked",
|
||||
reason,
|
||||
failureKind: "web-probe-final-memory-start-guard",
|
||||
jobId,
|
||||
stateDir,
|
||||
observerCreated: false,
|
||||
startBlocked: true,
|
||||
mutation: true,
|
||||
stateArtifactCreated: true,
|
||||
browserCreated: false,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
const launchFailureOutputSource = "const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const guard=read('launch-guard.json'); const readiness=read('launch-ready.json'); const pid=Number(process.argv[3]); const bool=(v)=>v==='true'; console.log(JSON.stringify({ok:false,command:'web-probe-observe start',status:'blocked',reason:'launch-readiness-unconfirmed',failureKind:'web-probe-launch-readiness',jobId:process.argv[2],stateDir:dir,observerCreated:false,browserCreated:false,browserCreationAttempted:true,startBlocked:true,mutation:true,stateArtifactCreated:true,memoryGuard:guard&&guard.memoryGuard||null,finalLaunchGuard:guard,launchReadiness:readiness,cleanup:{attempted:true,processGroupId:pid,termSent:bool(process.argv[4]),killSent:bool(process.argv[5]),aliveAfter:bool(process.argv[6]),completed:!bool(process.argv[6]),observedAt:new Date().toISOString(),valuesRedacted:true},valuesRedacted:true},null,2))";
|
||||
const cleanupPollCount = Math.ceil(finalMemoryPolicy.launchReadyTimeoutSeconds * 1000 / finalMemoryPolicy.launchReadyPollMilliseconds);
|
||||
const script = [
|
||||
"set -eu",
|
||||
`state_dir=${shellQuote(stateDir)}`,
|
||||
@@ -552,16 +718,71 @@ export function runNodeWebProbeObserveStart(
|
||||
"node -e \"const fs=require('fs'); fs.writeFileSync(process.argv[1], Buffer.from(fs.readFileSync(process.argv[2], 'utf8').replace(/\\s+/g, ''), 'base64'))\" \"$runner\" \"$runner_b64\"",
|
||||
"rm -f \"$runner_b64\"",
|
||||
"chmod 700 \"$runner\"",
|
||||
`if command -v setsid >/dev/null 2>&1; then setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & else nohup env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null & fi`,
|
||||
`launch_lock=${shellQuote(finalMemoryPolicy.launchLockPath)}`,
|
||||
`launch_guard_result="$state_dir/launch-guard.json"`,
|
||||
`launch_ready_result="$state_dir/launch-ready.json"`,
|
||||
`if ! command -v flock >/dev/null 2>&1; then printf '%s\\n' ${shellQuote(finalGuardFailure("host-flock-unavailable"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
|
||||
`if ! command -v setsid >/dev/null 2>&1; then printf '%s\\n' ${shellQuote(finalGuardFailure("host-setsid-unavailable"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
|
||||
`exec 9>"$launch_lock"`,
|
||||
`if ! flock -w ${shellQuote(String(finalMemoryPolicy.launchLockTimeoutSeconds))} 9; then printf '%s\\n' ${shellQuote(finalGuardFailure("launch-lock-timeout"))} >"$launch_guard_result"; cat "$launch_guard_result"; exit 0; fi`,
|
||||
"trap 'flock -u 9 2>/dev/null || true' EXIT",
|
||||
"set +e",
|
||||
`node - ${shellQuote(String(finalMemoryPolicy.thresholdBytes))} ${shellQuote(finalComparator)} ${shellQuote(options.memoryGuardMode)} ${shellQuote(jobId)} ${shellQuote(stateDir)} >"$launch_guard_result" <<'UNIDESK_WEB_OBSERVE_FINAL_GUARD'`,
|
||||
nodeWebObserveFinalLaunchGuardSource(),
|
||||
"UNIDESK_WEB_OBSERVE_FINAL_GUARD",
|
||||
"launch_guard_rc=$?",
|
||||
"set -e",
|
||||
"if [ \"$launch_guard_rc\" -ne 0 ]; then cat \"$launch_guard_result\"; flock -u 9; exit 0; fi",
|
||||
`setsid env ${runnerEnvAssignments} node "$runner" >"$state_dir/stdout.log" 2>"$state_dir/stderr.log" </dev/null 9>&- &`,
|
||||
"pid=$!",
|
||||
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
|
||||
"sleep 1",
|
||||
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
||||
"set +e",
|
||||
`node - "$pid" "$state_dir" ${shellQuote(jobId)} ${shellQuote(String(finalMemoryPolicy.launchReadyTimeoutSeconds * 1000))} ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds))} >"$launch_ready_result" <<'UNIDESK_WEB_OBSERVE_LAUNCH_READY'`,
|
||||
nodeWebObserveLaunchReadinessSource(),
|
||||
"UNIDESK_WEB_OBSERVE_LAUNCH_READY",
|
||||
"launch_ready_rc=$?",
|
||||
"set -e",
|
||||
`if [ "$launch_ready_rc" -ne 0 ]; then term_sent=false; if kill -TERM -- -"$pid" 2>/dev/null; then term_sent=true; fi; i=0; while [ "$i" -lt ${shellQuote(String(cleanupPollCount))} ] && kill -0 -- -"$pid" 2>/dev/null; do sleep ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds / 1000))}; i=$((i+1)); done; kill_sent=false; if kill -0 -- -"$pid" 2>/dev/null; then if kill -KILL -- -"$pid" 2>/dev/null; then kill_sent=true; fi; fi; i=0; while [ "$i" -lt ${shellQuote(String(cleanupPollCount))} ] && kill -0 -- -"$pid" 2>/dev/null; do sleep ${shellQuote(String(finalMemoryPolicy.launchReadyPollMilliseconds / 1000))}; i=$((i+1)); done; alive_after=false; if kill -0 -- -"$pid" 2>/dev/null; then alive_after=true; fi; node -e ${shellQuote(launchFailureOutputSource)} "$state_dir" ${shellQuote(jobId)} "$pid" "$term_sent" "$kill_sent" "$alive_after"; flock -u 9; trap - EXIT; exit 0; fi`,
|
||||
"flock -u 9",
|
||||
"trap - EXIT",
|
||||
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),finalLaunchGuard:read('launch-guard.json'),launchReadiness:read('launch-ready.json'),observerCreated:true,browserCreated:true,observerMutation:true,stateArtifactMutation:true,statusCommand:'bun scripts/cli.ts web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const startResolution = resolveWebObserveActionJson(result, "start");
|
||||
const started = startResolution.parsed;
|
||||
if (started?.observerCreated === false && started.startBlocked === true) {
|
||||
const finalSnapshot = record(started.memoryGuard);
|
||||
const finalMemoryGuard = evaluateWebProbeMemoryGuard(spec, options.memoryGuardMode, {
|
||||
ok: finalSnapshot.metric === "MemAvailable" && finalSnapshot.source === "/proc/meminfo" && finalSnapshot.swapExcluded === true,
|
||||
action: "gc remote memory",
|
||||
metric: finalSnapshot.metric,
|
||||
source: finalSnapshot.source,
|
||||
swapExcluded: finalSnapshot.swapExcluded,
|
||||
memory: { availableBytes: finalSnapshot.availableBytes },
|
||||
});
|
||||
return {
|
||||
...started,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
workspace: spec.workspace,
|
||||
initialMemoryGuard: memoryGuard,
|
||||
memoryGuard: finalMemoryGuard,
|
||||
finalLaunchGuard: started.finalLaunchGuard ?? started,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
const startOk = result.exitCode === 0 && started?.ok === true;
|
||||
const allowedFinalSnapshot = record(record(started?.finalLaunchGuard).memoryGuard);
|
||||
const effectiveMemoryGuard = allowedFinalSnapshot.metric === "MemAvailable"
|
||||
? evaluateWebProbeMemoryGuard(spec, options.memoryGuardMode, {
|
||||
ok: true,
|
||||
action: "gc remote memory",
|
||||
metric: allowedFinalSnapshot.metric,
|
||||
source: allowedFinalSnapshot.source,
|
||||
swapExcluded: allowedFinalSnapshot.swapExcluded,
|
||||
memory: { availableBytes: allowedFinalSnapshot.availableBytes },
|
||||
})
|
||||
: memoryGuard;
|
||||
const recovery = startOk
|
||||
? null
|
||||
: readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30));
|
||||
@@ -620,7 +841,10 @@ export function runNodeWebProbeObserveStart(
|
||||
alertThresholds,
|
||||
browserFreezePolicy,
|
||||
projectManagement,
|
||||
memoryGuard,
|
||||
initialMemoryGuard: memoryGuard,
|
||||
memoryGuard: effectiveMemoryGuard,
|
||||
finalLaunchGuard: started?.finalLaunchGuard ?? null,
|
||||
launchReadiness: started?.launchReadiness ?? null,
|
||||
targetPath: options.targetPath,
|
||||
id: observerId,
|
||||
degradedReason,
|
||||
|
||||
@@ -189,12 +189,28 @@ test("observe start inherits finite observer lifecycle limits from the owning YA
|
||||
assert.deepEqual(spec.webProbe?.resourcePolicy?.observerLifecycle, {
|
||||
maxRunSeconds: 3600,
|
||||
maxSamples: 720,
|
||||
maxScreenshots: 48,
|
||||
maxPerformanceEntries: 2000,
|
||||
maxScreenshots: 24,
|
||||
maxPerformanceEntries: 1000,
|
||||
maxBrowserProcessHistoryEntries: 360,
|
||||
maxBrowserFreezeSignalHistoryEntries: 120,
|
||||
maxRealtimeEventEntries: 256,
|
||||
maxRealtimeEventBodyBytes: 65536,
|
||||
maxDomEvidenceRows: 1000,
|
||||
maxProcessScanEntries: 8192,
|
||||
maxPerformanceCaptureSeconds: 30,
|
||||
maxResponseBodyBytes: 524288,
|
||||
maxWebPerformanceBodyBytes: 65536,
|
||||
maxJsonlQueueEntries: 512,
|
||||
maxJsonlQueueBytes: 4194304,
|
||||
maxJsonlLineBytes: 262144,
|
||||
jsonlFlushTimeoutMs: 5000,
|
||||
maxResponseBodyInFlight: 2,
|
||||
maxPassiveListenerTasks: 64,
|
||||
passiveTaskFlushTimeoutMs: 5000,
|
||||
closeStepTimeoutMs: 5000,
|
||||
maxProcessReadConcurrency: 32,
|
||||
maxRealtimeTotalEntries: 512,
|
||||
maxRealtimeTotalBodyBytes: 4194304,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user