Codex
717 lines
35 KiB
TypeScript
717 lines
35 KiB
TypeScript
// SPEC: PJ2026-01040111 长程观测 draft-2026-06-20-p0-passive-web-probe-observer.
|
|
// Responsibility: Source strings for the pure-client HWLAB web-probe observer and offline analyzer.
|
|
|
|
export function nodeWebObserveRunnerSource(): string {
|
|
return String.raw`#!/usr/bin/env node
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { appendFile, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
|
|
const specRef = "PJ2026-01040111 长程观测 draft-2026-06-20-p0-passive-web-probe-observer";
|
|
const startedAtMs = Date.now();
|
|
const startedAt = new Date(startedAtMs).toISOString();
|
|
const baseUrl = normalizeBaseUrl(process.env.HWLAB_WEB_BASE_URL);
|
|
const username = process.env.HWLAB_WEB_USER || "admin";
|
|
const password = process.env.HWLAB_WEB_PASS || "";
|
|
const stateDir = path.resolve(process.env.UNIDESK_WEB_OBSERVE_STATE_DIR || ".state/web-observe/manual");
|
|
const jobId = safeId(process.env.UNIDESK_WEB_OBSERVE_JOB_ID || "webobs-" + Date.now().toString(36) + "-" + randomBytes(3).toString("hex"));
|
|
const targetPath = process.env.UNIDESK_WEB_OBSERVE_TARGET_PATH || "/workbench";
|
|
const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS, 5000);
|
|
const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000);
|
|
const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0);
|
|
const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900");
|
|
const pageId = "page-" + randomBytes(4).toString("hex");
|
|
const dirs = {
|
|
commandsPending: path.join(stateDir, "commands", "pending"),
|
|
commandsProcessing: path.join(stateDir, "commands", "processing"),
|
|
commandsDone: path.join(stateDir, "commands", "done"),
|
|
commandsFailed: path.join(stateDir, "commands", "failed"),
|
|
screenshots: path.join(stateDir, "screenshots"),
|
|
analysis: path.join(stateDir, "analysis"),
|
|
};
|
|
const files = {
|
|
manifest: path.join(stateDir, "manifest.json"),
|
|
heartbeat: path.join(stateDir, "heartbeat.json"),
|
|
control: path.join(stateDir, "control.jsonl"),
|
|
samples: path.join(stateDir, "samples.jsonl"),
|
|
network: path.join(stateDir, "network.jsonl"),
|
|
console: path.join(stateDir, "console.jsonl"),
|
|
errors: path.join(stateDir, "errors.jsonl"),
|
|
artifacts: path.join(stateDir, "artifacts.jsonl"),
|
|
};
|
|
|
|
let browser;
|
|
let context;
|
|
let page;
|
|
let sampleSeq = 0;
|
|
let commandSeq = 0;
|
|
let artifactSeq = 0;
|
|
let activeCommandId = null;
|
|
let stopping = false;
|
|
let terminalStatus = "starting";
|
|
let lastScreenshotAtMs = 0;
|
|
let auth = null;
|
|
|
|
try {
|
|
if (!password) throw new Error("missing HWLAB_WEB_PASS");
|
|
await prepareDirs();
|
|
await writeManifest({ status: "starting" });
|
|
await writeHeartbeat({ status: "starting" });
|
|
const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
|
|
const { chromium } = await launcher.importPlaywright();
|
|
browser = await launcher.launchChromium(chromium);
|
|
context = await browser.newContext({ viewport });
|
|
auth = await runControlCommand({ id: "startup-login", type: "login", createdAt: startedAt, source: "startup" }, async () => authenticate(context));
|
|
page = await context.newPage();
|
|
attachPassiveListeners(page);
|
|
await runControlCommand({ id: "startup-goto", type: "goto", path: targetPath, createdAt: new Date().toISOString(), source: "startup" }, async () => gotoTarget(targetPath));
|
|
terminalStatus = "running";
|
|
await writeManifest({ status: "running", auth: publicAuth(auth) });
|
|
await writeHeartbeat({ status: "running" });
|
|
while (!stopping) {
|
|
await drainOneCommand();
|
|
await samplePage("interval");
|
|
if (maxSamples > 0 && sampleSeq >= maxSamples) {
|
|
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
|
|
break;
|
|
}
|
|
await sleep(sampleIntervalMs);
|
|
}
|
|
terminalStatus = "completed";
|
|
await writeHeartbeat({ status: "completed" });
|
|
await writeManifest({ status: "completed", completedAt: new Date().toISOString() });
|
|
process.exitCode = 0;
|
|
} catch (error) {
|
|
terminalStatus = "failed";
|
|
await appendJsonl(files.errors, eventRecord("runner-error", { error: errorSummary(error) })).catch(() => {});
|
|
await writeHeartbeat({ status: "failed", error: errorSummary(error) }).catch(() => {});
|
|
await writeManifest({ status: "failed", error: errorSummary(error) }).catch(() => {});
|
|
process.exitCode = 2;
|
|
} finally {
|
|
if (browser) await browser.close().catch(() => {});
|
|
}
|
|
|
|
async function prepareDirs() {
|
|
await mkdir(stateDir, { recursive: true, mode: 0o700 });
|
|
await Promise.all(Object.values(dirs).map((dir) => mkdir(dir, { recursive: true, mode: 0o700 })));
|
|
}
|
|
|
|
async function writeManifest(extra = {}) {
|
|
const manifest = {
|
|
ok: extra.status !== "failed",
|
|
command: "web-probe-observe",
|
|
specRef,
|
|
jobId,
|
|
pid: process.pid,
|
|
stateDir,
|
|
baseUrl,
|
|
targetPath,
|
|
pageAuthority: { browser: "chromium", context: "single", pageId, continuityBreaksRecorded: true },
|
|
sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerInitiatedDefault: false, responseBodyReadDefault: false },
|
|
commandDirs: dirs,
|
|
artifacts: files,
|
|
safety: { pureClient: true, inboundApi: false, database: false, queueConsumer: false, k8sWorkload: false, valuesRedacted: true, secretValuesPrinted: false },
|
|
startedAt,
|
|
...extra,
|
|
};
|
|
await writeFile(files.manifest, JSON.stringify(manifest, null, 2) + "\n", { mode: 0o600 });
|
|
}
|
|
|
|
async function writeHeartbeat(extra = {}) {
|
|
const heartbeat = {
|
|
ok: terminalStatus !== "failed",
|
|
jobId,
|
|
pid: process.pid,
|
|
stateDir,
|
|
status: terminalStatus,
|
|
pageId,
|
|
baseUrl,
|
|
currentUrl: currentPageUrl(),
|
|
sampleSeq,
|
|
commandSeq,
|
|
activeCommandId,
|
|
updatedAt: new Date().toISOString(),
|
|
uptimeMs: Date.now() - startedAtMs,
|
|
...extra,
|
|
};
|
|
await writeFile(files.heartbeat, JSON.stringify(heartbeat, null, 2) + "\n", { mode: 0o600 });
|
|
}
|
|
|
|
function attachPassiveListeners(targetPage) {
|
|
targetPage.on("request", (request) => {
|
|
void appendJsonl(files.network, eventRecord("request", {
|
|
observerInitiated: false,
|
|
commandId: activeCommandId,
|
|
method: request.method(),
|
|
url: safeUrl(request.url()),
|
|
resourceType: request.resourceType(),
|
|
frameUrl: safeFrameUrl(request.frame()),
|
|
}));
|
|
});
|
|
targetPage.on("response", (response) => {
|
|
const request = response.request();
|
|
void appendJsonl(files.network, eventRecord("response", {
|
|
observerInitiated: false,
|
|
commandId: activeCommandId,
|
|
method: request.method(),
|
|
url: safeUrl(response.url()),
|
|
resourceType: request.resourceType(),
|
|
status: response.status(),
|
|
statusText: response.statusText(),
|
|
fromServiceWorker: response.fromServiceWorker(),
|
|
bodyRead: false,
|
|
}));
|
|
});
|
|
targetPage.on("requestfailed", (request) => {
|
|
void appendJsonl(files.network, eventRecord("requestfailed", {
|
|
observerInitiated: false,
|
|
commandId: activeCommandId,
|
|
method: request.method(),
|
|
url: safeUrl(request.url()),
|
|
resourceType: request.resourceType(),
|
|
failure: request.failure()?.errorText ?? null,
|
|
}));
|
|
});
|
|
targetPage.on("console", (message) => {
|
|
void appendJsonl(files.console, eventRecord("console", { type: message.type(), text: truncate(message.text(), 1000), location: message.location() }));
|
|
});
|
|
targetPage.on("pageerror", (error) => {
|
|
void appendJsonl(files.errors, eventRecord("pageerror", { error: errorSummary(error) }));
|
|
});
|
|
targetPage.on("crash", () => {
|
|
void appendJsonl(files.errors, eventRecord("page-crash", { pageId }));
|
|
});
|
|
targetPage.on("close", () => {
|
|
void appendJsonl(files.control, eventRecord("continuity-break", { pageId, reason: "page-closed" }));
|
|
});
|
|
}
|
|
|
|
async function drainOneCommand() {
|
|
const entries = (await readdir(dirs.commandsPending).catch(() => [])).filter((name) => name.endsWith(".json")).sort();
|
|
const name = entries[0];
|
|
if (!name) return;
|
|
const pending = path.join(dirs.commandsPending, name);
|
|
const processing = path.join(dirs.commandsProcessing, name);
|
|
await rename(pending, processing).catch(() => null);
|
|
const raw = await readFile(processing, "utf8");
|
|
const command = JSON.parse(raw);
|
|
const id = safeId(command.id || name.replace(/[.]json$/u, ""));
|
|
command.id = id;
|
|
try {
|
|
const result = await processCommand(command);
|
|
const done = { ok: true, commandId: id, type: command.type, completedAt: new Date().toISOString(), result: sanitize(result) };
|
|
await writeFile(path.join(dirs.commandsDone, id + ".json"), JSON.stringify(done, null, 2) + "\n", { mode: 0o600 });
|
|
await appendJsonl(files.control, controlRecord(command, "completed", done.result));
|
|
await unlink(processing).catch(() => {});
|
|
} catch (error) {
|
|
const failed = { ok: false, commandId: id, type: command.type, failedAt: new Date().toISOString(), error: errorSummary(error) };
|
|
await writeFile(path.join(dirs.commandsFailed, id + ".json"), JSON.stringify(failed, null, 2) + "\n", { mode: 0o600 });
|
|
await appendJsonl(files.control, controlRecord(command, "failed", failed.error));
|
|
await unlink(processing).catch(() => {});
|
|
} finally {
|
|
activeCommandId = null;
|
|
await writeHeartbeat({ status: terminalStatus });
|
|
}
|
|
}
|
|
|
|
async function processCommand(command) {
|
|
commandSeq += 1;
|
|
activeCommandId = command.id;
|
|
await writeHeartbeat({ status: "running", activeCommandId });
|
|
await appendJsonl(files.control, controlRecord(command, "started", commandInputSummary(command)));
|
|
switch (command.type) {
|
|
case "login": return authenticate(context);
|
|
case "preflight": return preflightSummary();
|
|
case "goto": return gotoTarget(command.path || command.url || targetPath);
|
|
case "sendPrompt": return sendPrompt(String(command.text || ""));
|
|
case "clickSession": return clickSession(String(command.sessionId || command.value || ""));
|
|
case "screenshot": return captureScreenshot(command.reason || "manual", command.imageType || "png");
|
|
case "mark": return { mark: truncate(command.label || command.text || "mark", 200), currentUrl: currentPageUrl(), pageId };
|
|
case "stop": stopping = true; return { stopping: true, currentUrl: currentPageUrl(), pageId };
|
|
default: throw new Error("unsupported observer command type: " + command.type);
|
|
}
|
|
}
|
|
|
|
async function runControlCommand(command, fn) {
|
|
activeCommandId = command.id;
|
|
commandSeq += 1;
|
|
const beforeUrl = currentPageUrl();
|
|
const started = Date.now();
|
|
await appendJsonl(files.control, controlRecord(command, "started", { beforeUrl, input: commandInputSummary(command) }));
|
|
try {
|
|
const result = await fn();
|
|
await appendJsonl(files.control, controlRecord(command, "completed", { beforeUrl, afterUrl: currentPageUrl(), durationMs: Date.now() - started, result: sanitize(result) }));
|
|
return result;
|
|
} catch (error) {
|
|
await appendJsonl(files.control, controlRecord(command, "failed", { beforeUrl, afterUrl: currentPageUrl(), durationMs: Date.now() - started, error: errorSummary(error) }));
|
|
throw error;
|
|
} finally {
|
|
activeCommandId = null;
|
|
}
|
|
}
|
|
|
|
async function authenticate(browserContext) {
|
|
const loginUrl = new URL("/auth/login", baseUrl).toString();
|
|
const response = await browserContext.request.post(loginUrl, {
|
|
data: { username, password },
|
|
headers: { accept: "application/json", "content-type": "application/json" },
|
|
timeout: 15000,
|
|
});
|
|
const cookies = await browserContext.cookies(baseUrl);
|
|
const cookieNames = cookies.map((cookie) => cookie.name).filter((name) => /session|auth|token/iu.test(name));
|
|
return {
|
|
ok: response.ok() && cookieNames.length > 0,
|
|
method: "api",
|
|
loginPath: new URL(loginUrl).pathname,
|
|
status: response.status(),
|
|
statusText: response.statusText(),
|
|
cookiePresent: cookieNames.length > 0,
|
|
cookieNames,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function publicAuth(value) {
|
|
if (!value) return null;
|
|
return { ok: value.ok === true, method: value.method, status: value.status, cookiePresent: value.cookiePresent === true, cookieNames: value.cookieNames || [], valuesRedacted: true };
|
|
}
|
|
|
|
async function gotoTarget(rawTarget) {
|
|
const target = new URL(String(rawTarget || targetPath), baseUrl).toString();
|
|
const beforeUrl = currentPageUrl();
|
|
const response = await page.goto(target, { waitUntil: "domcontentloaded", timeout: 30000 });
|
|
await page.waitForTimeout(1000).catch(() => {});
|
|
return { beforeUrl, afterUrl: currentPageUrl(), httpStatus: response ? response.status() : null, pageId };
|
|
}
|
|
|
|
async function sendPrompt(text) {
|
|
if (text.trim().length === 0) throw new Error("sendPrompt requires non-empty text");
|
|
const beforeUrl = currentPageUrl();
|
|
const editor = page.locator('textarea, [role="textbox"], [contenteditable="true"], input[type="text"]').last();
|
|
await editor.waitFor({ state: "visible", timeout: 15000 });
|
|
const tag = await editor.evaluate((element) => element.tagName.toLowerCase()).catch(() => "");
|
|
const editable = await editor.evaluate((element) => element.getAttribute("contenteditable") === "true").catch(() => false);
|
|
if (tag === "textarea" || tag === "input") await editor.fill(text);
|
|
else if (editable) {
|
|
await editor.click();
|
|
await page.keyboard.insertText(text);
|
|
} else {
|
|
await editor.click();
|
|
await page.keyboard.insertText(text);
|
|
}
|
|
const submit = page.locator([
|
|
'button[type="submit"]',
|
|
'button:has-text("发送")',
|
|
'button:has-text("Send")',
|
|
'[data-testid*="send" i]',
|
|
'[aria-label*="send" i]',
|
|
'[aria-label*="发送"]'
|
|
].join(", ")).last();
|
|
await submit.waitFor({ state: "visible", timeout: 15000 });
|
|
await submit.click();
|
|
await page.waitForTimeout(1500);
|
|
return { beforeUrl, afterUrl: currentPageUrl(), textHash: sha256Text(text), textBytes: Buffer.byteLength(text), pageId };
|
|
}
|
|
|
|
async function clickSession(sessionId) {
|
|
if (!sessionId) throw new Error("clickSession requires --session-id or --value");
|
|
const beforeUrl = currentPageUrl();
|
|
const escaped = cssEscape(sessionId);
|
|
const candidate = page.locator("[data-session-id=\"" + escaped + "\"], [href*=\"" + escaped + "\"], text=" + sessionId).first();
|
|
await candidate.waitFor({ state: "visible", timeout: 15000 });
|
|
await candidate.click();
|
|
await page.waitForTimeout(1000);
|
|
return { beforeUrl, afterUrl: currentPageUrl(), sessionId, pageId };
|
|
}
|
|
|
|
async function preflightSummary() {
|
|
return { currentUrl: currentPageUrl(), title: await page.title().catch(() => null), pageId, auth: publicAuth(auth) };
|
|
}
|
|
|
|
async function samplePage(reason) {
|
|
if (!page || page.isClosed()) return;
|
|
sampleSeq += 1;
|
|
const dom = await page.evaluate(() => {
|
|
const trim = (value, limit = 500) => String(value || "").replace(/\s+/g, " ").trim().slice(0, limit);
|
|
const visible = (element) => {
|
|
const rect = element.getBoundingClientRect();
|
|
const style = window.getComputedStyle(element);
|
|
return rect.width > 0 && rect.height > 0 && style.visibility !== "hidden" && style.display !== "none";
|
|
};
|
|
const textHashInput = (element) => trim(element.textContent || "", 800);
|
|
const summarize = (selector, limit) => Array.from(document.querySelectorAll(selector)).filter(visible).slice(-limit).map((element, index) => {
|
|
const rect = element.getBoundingClientRect();
|
|
return {
|
|
index,
|
|
tag: element.tagName.toLowerCase(),
|
|
testId: element.getAttribute("data-testid"),
|
|
role: element.getAttribute("role"),
|
|
status: element.getAttribute("data-status") || element.getAttribute("aria-busy") || null,
|
|
text: textHashInput(element),
|
|
rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
|
|
};
|
|
});
|
|
const url = location.href;
|
|
const routeSessionMatch = url.match(/\/workbench\/sessions\/([^/?#]+)/u);
|
|
const activeSession = document.querySelector('[data-active="true"][data-session-id], [aria-selected="true"][data-session-id], .active[data-session-id]');
|
|
const activeSessionId = activeSession ? activeSession.getAttribute("data-session-id") : null;
|
|
const messageSelector = '[data-testid*="message" i], [class*="message" i], article, [role="article"]';
|
|
const traceSelector = '[data-testid*="trace" i], [class*="trace" i], [data-trace-id], [data-testid*="event" i]';
|
|
const messages = summarize(messageSelector, 20);
|
|
const traceRows = summarize(traceSelector, 30);
|
|
const active = document.activeElement;
|
|
return {
|
|
url,
|
|
path: location.pathname,
|
|
routeSessionId: routeSessionMatch ? decodeURIComponent(routeSessionMatch[1]) : null,
|
|
activeSessionId,
|
|
title: document.title,
|
|
focus: active ? { tag: active.tagName.toLowerCase(), testId: active.getAttribute("data-testid"), role: active.getAttribute("role") } : null,
|
|
viewport: { width: window.innerWidth, height: window.innerHeight, devicePixelRatio: window.devicePixelRatio },
|
|
scroll: { x: Math.round(window.scrollX), y: Math.round(window.scrollY), height: Math.round(document.documentElement.scrollHeight), width: Math.round(document.documentElement.scrollWidth) },
|
|
messages,
|
|
traceRows,
|
|
performance: performance.getEntriesByType("resource").slice(-30).map((entry) => ({ name: entry.name.split(/[?#]/u)[0].slice(0, 240), initiatorType: entry.initiatorType, startTime: Math.round(entry.startTime), duration: Math.round(entry.duration) })),
|
|
};
|
|
}).catch((error) => ({ error: errorSummary(error), url: currentPageUrl() }));
|
|
const sample = {
|
|
seq: sampleSeq,
|
|
ts: new Date().toISOString(),
|
|
reason,
|
|
pageId,
|
|
commandId: activeCommandId,
|
|
observerInitiated: false,
|
|
...digestDom(dom),
|
|
};
|
|
await appendJsonl(files.samples, sample);
|
|
if (screenshotIntervalMs > 0 && Date.now() - lastScreenshotAtMs >= screenshotIntervalMs) {
|
|
await captureScreenshot("checkpoint", "jpeg").catch((error) => appendJsonl(files.errors, eventRecord("screenshot-error", { error: errorSummary(error) })));
|
|
}
|
|
await writeHeartbeat({ status: terminalStatus });
|
|
}
|
|
|
|
function digestDom(dom) {
|
|
if (dom && dom.error) return dom;
|
|
const messages = Array.isArray(dom.messages) ? dom.messages.map((item) => ({ ...item, textHash: sha256Text(item.text || ""), textPreview: truncate(item.text || "", 160), textBytes: Buffer.byteLength(item.text || "") })) : [];
|
|
const traceRows = Array.isArray(dom.traceRows) ? dom.traceRows.map((item) => ({ ...item, textHash: sha256Text(item.text || ""), textPreview: truncate(item.text || "", 160), textBytes: Buffer.byteLength(item.text || "") })) : [];
|
|
return { ...dom, messages, traceRows };
|
|
}
|
|
|
|
async function captureScreenshot(reason, imageType = "png") {
|
|
if (!page || page.isClosed()) throw new Error("page is not available for screenshot");
|
|
artifactSeq += 1;
|
|
const safeReason = safeId(String(reason || "manual")).slice(0, 40) || "manual";
|
|
const type = imageType === "jpeg" || imageType === "jpg" ? "jpeg" : "png";
|
|
const ext = type === "jpeg" ? "jpg" : "png";
|
|
const file = path.join(dirs.screenshots, String(sampleSeq).padStart(6, "0") + "_" + String(artifactSeq).padStart(4, "0") + "_" + safeReason + "." + ext);
|
|
const options = type === "jpeg" ? { path: file, type: "jpeg", quality: 70, fullPage: false } : { path: file, type: "png", fullPage: false };
|
|
await page.screenshot(options);
|
|
const meta = await fileMeta(file);
|
|
const artifact = { seq: artifactSeq, sampleSeq, ts: new Date().toISOString(), kind: "screenshot", reason, path: file, type, byteCount: meta.byteCount, sha256: meta.sha256, pageId, currentUrl: currentPageUrl() };
|
|
await appendJsonl(files.artifacts, artifact);
|
|
lastScreenshotAtMs = Date.now();
|
|
return artifact;
|
|
}
|
|
|
|
function eventRecord(type, data) {
|
|
return { ts: new Date().toISOString(), type, jobId, pageId, sampleSeq, commandId: activeCommandId, ...sanitize(data) };
|
|
}
|
|
|
|
function controlRecord(command, phase, detail) {
|
|
return {
|
|
ts: new Date().toISOString(),
|
|
seq: commandSeq,
|
|
phase,
|
|
commandId: command.id,
|
|
type: command.type,
|
|
source: command.source || "file",
|
|
input: commandInputSummary(command),
|
|
beforeUrl: command.beforeUrl || null,
|
|
afterUrl: currentPageUrl(),
|
|
pageId,
|
|
detail: sanitize(detail),
|
|
};
|
|
}
|
|
|
|
function commandInputSummary(command) {
|
|
const text = typeof command.text === "string" ? command.text : null;
|
|
return {
|
|
type: command.type,
|
|
path: command.path || null,
|
|
url: command.url ? safeUrl(command.url) : null,
|
|
sessionId: command.sessionId || command.value || null,
|
|
label: command.label ? truncate(command.label, 200) : null,
|
|
textHash: text === null ? null : sha256Text(text),
|
|
textBytes: text === null ? null : Buffer.byteLength(text),
|
|
textPreview: null,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
async function appendJsonl(file, value) {
|
|
await appendFile(file, JSON.stringify(sanitize(value)) + "\n", { mode: 0o600 });
|
|
}
|
|
|
|
async function fileMeta(file) {
|
|
const [buffer, stats] = await Promise.all([readFile(file), stat(file)]);
|
|
return { byteCount: stats.size, sha256: "sha256:" + createHash("sha256").update(buffer).digest("hex") };
|
|
}
|
|
|
|
function currentPageUrl() {
|
|
try { return page && !page.isClosed() ? page.url() : null; } catch { return null; }
|
|
}
|
|
|
|
function safeFrameUrl(frame) {
|
|
try { return frame ? safeUrl(frame.url()) : null; } catch { return null; }
|
|
}
|
|
|
|
function safeUrl(value) {
|
|
try {
|
|
const url = new URL(String(value), baseUrl);
|
|
for (const key of Array.from(url.searchParams.keys())) {
|
|
if (/token|key|secret|password|auth|cookie/iu.test(key)) url.searchParams.set(key, "[redacted]");
|
|
}
|
|
return url.toString();
|
|
} catch {
|
|
return truncate(String(value || ""), 300);
|
|
}
|
|
}
|
|
|
|
function normalizeBaseUrl(value) {
|
|
const raw = value || "http://127.0.0.1:3000";
|
|
const url = new URL(raw);
|
|
return url.origin;
|
|
}
|
|
|
|
function parseViewport(value) {
|
|
const match = String(value).match(/^(\d{3,5})x(\d{3,5})$/u);
|
|
return match ? { width: Number(match[1]), height: Number(match[2]) } : { width: 1440, height: 900 };
|
|
}
|
|
|
|
function positiveInteger(value, fallback) {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallback;
|
|
}
|
|
|
|
function sha256Text(value) {
|
|
return "sha256:" + createHash("sha256").update(String(value)).digest("hex");
|
|
}
|
|
|
|
function safeId(value) {
|
|
return String(value || "").replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "").slice(0, 120) || "item";
|
|
}
|
|
|
|
function cssEscape(value) {
|
|
return String(value).replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"");
|
|
}
|
|
|
|
function truncate(value, limit) {
|
|
const text = String(value || "");
|
|
return text.length > limit ? text.slice(0, limit) + "..." : text;
|
|
}
|
|
|
|
function sanitize(value) {
|
|
if (value === null || value === undefined) return value;
|
|
if (typeof value === "string") return value === password ? "[redacted]" : value.replaceAll(password, "[redacted]");
|
|
if (typeof value === "number" || typeof value === "boolean") return value;
|
|
if (Array.isArray(value)) return value.map(sanitize);
|
|
if (typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => /password|cookie|authorization|token|secret/iu.test(key) ? [key, "[redacted]"] : [key, sanitize(item)]));
|
|
return String(value);
|
|
}
|
|
|
|
function errorSummary(error) {
|
|
return { name: error && error.name ? error.name : "Error", message: error && error.message ? truncate(error.message, 1000) : truncate(String(error), 1000), stackTail: error && error.stack ? truncate(error.stack, 2000) : null };
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
|
|
}
|
|
`;
|
|
}
|
|
|
|
export function nodeWebObserveAnalyzerSource(): string {
|
|
return String.raw`#!/usr/bin/env node
|
|
import { createHash } from "node:crypto";
|
|
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
|
|
const stateDir = path.resolve(process.env.UNIDESK_WEB_OBSERVE_STATE_DIR || process.argv[2] || ".state/web-observe/manual");
|
|
const analysisDir = path.join(stateDir, "analysis");
|
|
const reportJsonPath = path.join(analysisDir, "report.json");
|
|
const reportMdPath = path.join(analysisDir, "report.md");
|
|
const samples = await readJsonl(path.join(stateDir, "samples.jsonl"));
|
|
const control = await readJsonl(path.join(stateDir, "control.jsonl"));
|
|
const network = await readJsonl(path.join(stateDir, "network.jsonl"));
|
|
const errors = await readJsonl(path.join(stateDir, "errors.jsonl"));
|
|
const artifacts = await readJsonl(path.join(stateDir, "artifacts.jsonl"));
|
|
const manifest = await readJson(path.join(stateDir, "manifest.json"));
|
|
const heartbeat = await readJson(path.join(stateDir, "heartbeat.json"));
|
|
|
|
await mkdir(analysisDir, { recursive: true, mode: 0o700 });
|
|
const findings = buildFindings(samples, control, network, errors);
|
|
const transitions = buildTransitions(samples);
|
|
const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl }));
|
|
const report = {
|
|
ok: findings.filter((item) => item.severity === "red").length === 0,
|
|
command: "web-probe-observe analyze",
|
|
generatedAt: new Date().toISOString(),
|
|
stateDir,
|
|
manifest: compactManifest(manifest),
|
|
heartbeat: compactHeartbeat(heartbeat),
|
|
counts: { samples: samples.length, control: control.length, network: network.length, errors: errors.length, artifacts: artifacts.length },
|
|
commandTimeline,
|
|
transitions,
|
|
findings,
|
|
artifactSummary: await artifactSummary(artifacts),
|
|
safety: { offlineOnly: true, browserDriven: false, apiFetch: false, valuesRedacted: true },
|
|
};
|
|
await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 });
|
|
await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 });
|
|
const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]);
|
|
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
|
|
|
|
async function readJson(file) {
|
|
try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; }
|
|
}
|
|
|
|
async function readJsonl(file) {
|
|
try {
|
|
const text = await readFile(file, "utf8");
|
|
return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => {
|
|
try { return JSON.parse(line); } catch { return { parseError: true, rawHash: sha256(line) }; }
|
|
});
|
|
} catch { return []; }
|
|
}
|
|
|
|
function buildFindings(samples, control, network, errors) {
|
|
const findings = [];
|
|
const commandTimes = control.filter((item) => item.phase === "completed" || item.phase === "started").map((item) => Date.parse(item.ts)).filter(Number.isFinite);
|
|
const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean));
|
|
const activeSessions = new Set(samples.map((item) => item.activeSessionId).filter(Boolean));
|
|
if (routeSessions.size > 1) findings.push({ id: "session-route-changed", severity: "amber", summary: "route session changed during observation", routeSessionCount: routeSessions.size, samples: sampleRefs(samples, (item) => item.routeSessionId) });
|
|
if (activeSessions.size > 1) findings.push({ id: "active-session-changed", severity: "amber", summary: "active session changed during observation", activeSessionCount: activeSessions.size, samples: sampleRefs(samples, (item) => item.activeSessionId) });
|
|
const mismatches = samples.filter((item) => item.routeSessionId && item.activeSessionId && item.routeSessionId !== item.activeSessionId);
|
|
if (mismatches.length > 0) findings.push({ id: "route-active-session-mismatch", severity: "red", summary: "routeSessionId and activeSessionId diverged", count: mismatches.length, samples: mismatches.slice(0, 10).map(ref) });
|
|
const uncommandedChanges = [];
|
|
for (let i = 1; i < samples.length; i += 1) {
|
|
const prev = digestSample(samples[i - 1]);
|
|
const next = digestSample(samples[i]);
|
|
if (prev !== next && !nearCommand(samples[i], commandTimes, 10000)) uncommandedChanges.push(ref(samples[i]));
|
|
}
|
|
if (uncommandedChanges.length > 0) findings.push({ id: "uncommanded-visible-state-change", severity: "amber", summary: "visible message/trace digest changed without a nearby command", count: uncommandedChanges.length, samples: uncommandedChanges.slice(0, 20) });
|
|
const finalFlicker = detectFinalFlicker(samples);
|
|
if (finalFlicker.length > 0) findings.push({ id: "final-response-flicker", severity: "red", summary: "message text digest disappeared or switched to diagnostic-like text after non-empty final text", count: finalFlicker.length, samples: finalFlicker.slice(0, 20) });
|
|
const scrollJumps = [];
|
|
for (let i = 1; i < samples.length; i += 1) {
|
|
const prevY = Number(samples[i - 1]?.scroll?.y ?? 0);
|
|
const nextY = Number(samples[i]?.scroll?.y ?? 0);
|
|
if (prevY > 250 && nextY < 40 && !nearCommand(samples[i], commandTimes, 8000)) scrollJumps.push({ from: ref(samples[i - 1]), to: ref(samples[i]) });
|
|
}
|
|
if (scrollJumps.length > 0) findings.push({ id: "scroll-jump-top", severity: "amber", summary: "scroll position jumped near top without nearby command", count: scrollJumps.length, samples: scrollJumps.slice(0, 10) });
|
|
const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => /completed|failed|canceled|terminal|done/iu.test((row.status || "") + " " + (row.textPreview || ""))));
|
|
const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0);
|
|
if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) });
|
|
const naturalApi = network.filter((item) => item.observerInitiated === false && item.type === "response" && /\/v1\/|\/auth\//u.test(String(item.url || "")));
|
|
findings.push({ id: "natural-api-dom-lag-baseline", severity: "info", summary: "natural API responses and DOM samples are available for manual lag correlation", naturalApiResponses: naturalApi.length, sampleCount: samples.length });
|
|
if (errors.length > 0) findings.push({ id: "browser-console-or-page-errors", severity: "amber", summary: "pageerror/runner errors were captured", count: errors.length, first: errors.slice(0, 5) });
|
|
if (samples.length === 0) findings.push({ id: "no-samples", severity: "red", summary: "observer produced no samples" });
|
|
return findings;
|
|
}
|
|
|
|
function buildTransitions(samples) {
|
|
const rows = [];
|
|
let last = null;
|
|
for (const sample of samples) {
|
|
const digest = digestSample(sample);
|
|
if (digest !== last) {
|
|
rows.push({ seq: sample.seq, ts: sample.ts, url: sample.url, routeSessionId: sample.routeSessionId || null, activeSessionId: sample.activeSessionId || null, messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0, traceRowCount: Array.isArray(sample.traceRows) ? sample.traceRows.length : 0, digest });
|
|
last = digest;
|
|
}
|
|
}
|
|
return rows.slice(0, 200);
|
|
}
|
|
|
|
function detectFinalFlicker(samples) {
|
|
const flickers = [];
|
|
let lastNonEmpty = null;
|
|
for (const sample of samples) {
|
|
const messageText = Array.isArray(sample.messages) ? sample.messages.map((item) => item.textPreview || "").join("\n") : "";
|
|
const nonEmpty = messageText.trim().length > 0;
|
|
if (nonEmpty && !/temporarily|timeout|无法连接|暂时|error|failed|超时/iu.test(messageText)) lastNonEmpty = { sample, messageText };
|
|
if (lastNonEmpty && nonEmpty && /temporarily|timeout|无法连接|暂时|error|failed|超时/iu.test(messageText)) flickers.push({ from: ref(lastNonEmpty.sample), to: ref(sample) });
|
|
if (lastNonEmpty && !nonEmpty) flickers.push({ from: ref(lastNonEmpty.sample), to: ref(sample), reason: "text-disappeared" });
|
|
}
|
|
return flickers;
|
|
}
|
|
|
|
function digestSample(sample) {
|
|
const messages = Array.isArray(sample.messages) ? sample.messages.map((item) => item.textHash || item.textPreview || "").join("|") : "";
|
|
const trace = Array.isArray(sample.traceRows) ? sample.traceRows.map((item) => (item.status || "") + ":" + (item.textHash || item.textPreview || "")).join("|") : "";
|
|
return sha256((sample.routeSessionId || "") + "|" + (sample.activeSessionId || "") + "|" + messages + "|" + trace);
|
|
}
|
|
|
|
function nearCommand(sample, commandTimes, windowMs) {
|
|
const ts = Date.parse(sample.ts);
|
|
return Number.isFinite(ts) && commandTimes.some((item) => Math.abs(ts - item) <= windowMs);
|
|
}
|
|
|
|
function sampleRefs(samples, pick) {
|
|
const seen = new Set();
|
|
const refs = [];
|
|
for (const sample of samples) {
|
|
const value = pick(sample);
|
|
if (!value || seen.has(value)) continue;
|
|
seen.add(value);
|
|
refs.push({ ...ref(sample), value });
|
|
}
|
|
return refs.slice(0, 20);
|
|
}
|
|
|
|
function ref(sample) {
|
|
if (!sample) return null;
|
|
return { seq: sample.seq ?? null, ts: sample.ts ?? null, url: sample.url ?? null, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null };
|
|
}
|
|
|
|
async function artifactSummary(artifacts) {
|
|
const items = artifacts.slice(-30).map((item) => ({ kind: item.kind, reason: item.reason, sampleSeq: item.sampleSeq, path: item.path, sha256: item.sha256, byteCount: item.byteCount }));
|
|
return { count: artifacts.length, latest: items };
|
|
}
|
|
|
|
function compactManifest(value) {
|
|
if (!value) return null;
|
|
return { jobId: value.jobId, stateDir: value.stateDir, baseUrl: value.baseUrl, targetPath: value.targetPath, startedAt: value.startedAt, status: value.status, sampling: value.sampling, safety: value.safety };
|
|
}
|
|
|
|
function compactHeartbeat(value) {
|
|
if (!value) return null;
|
|
return { jobId: value.jobId, pid: value.pid, status: value.status, sampleSeq: value.sampleSeq, commandSeq: value.commandSeq, currentUrl: value.currentUrl, updatedAt: value.updatedAt, uptimeMs: value.uptimeMs };
|
|
}
|
|
|
|
function renderMarkdown(report) {
|
|
const findingLines = report.findings.length === 0 ? "- 无红灯项。" : report.findings.map((item) => "- " + item.severity + ": " + item.id + " - " + item.summary).join("\n");
|
|
const commandLines = report.commandTimeline.length === 0 ? "- 无控制命令。" : report.commandTimeline.map((item) => "- " + item.ts + " " + item.phase + " " + item.type + " " + item.commandId + " " + (item.afterUrl || "")).join("\n");
|
|
const transitionLines = report.transitions.length === 0 ? "- 无状态变化。" : report.transitions.slice(0, 80).map((item) => "- #" + item.seq + " " + item.ts + " messages=" + item.messageCount + " traceRows=" + item.traceRowCount + " route=" + (item.routeSessionId || "-") + " active=" + (item.activeSessionId || "-")).join("\n");
|
|
return "# web-probe observe analysis\n\n"
|
|
+ "- stateDir: " + report.stateDir + "\n"
|
|
+ "- generatedAt: " + report.generatedAt + "\n"
|
|
+ "- samples: " + report.counts.samples + "\n"
|
|
+ "- control: " + report.counts.control + "\n"
|
|
+ "- network: " + report.counts.network + "\n"
|
|
+ "- errors: " + report.counts.errors + "\n\n"
|
|
+ "## Findings\n\n" + findingLines + "\n\n"
|
|
+ "## Command timeline\n\n" + commandLines + "\n\n"
|
|
+ "## State transitions\n\n" + transitionLines + "\n";
|
|
}
|
|
|
|
async function fileMeta(file) {
|
|
const [buffer, stats] = await Promise.all([readFile(file), stat(file)]);
|
|
return { byteCount: stats.size, sha256: "sha256:" + createHash("sha256").update(buffer).digest("hex") };
|
|
}
|
|
|
|
function sha256(value) {
|
|
return "sha256:" + createHash("sha256").update(String(value)).digest("hex");
|
|
}
|
|
`;
|
|
}
|