Merge pull request #527 from pikasTech/feat/web-probe-observe-519

feat: add passive web-probe observer
This commit is contained in:
Lyon
2026-06-20 23:22:43 +08:00
committed by GitHub
3 changed files with 1148 additions and 3 deletions
+8
View File
@@ -67,15 +67,23 @@ export function hwlabNodeWebProbeHelp(): Record<string, unknown> {
"bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --wait-messages-ms 1000",
"bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --url https://hwlab.pikapython.com --fresh-session --message 'ping'",
"bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 --script-file .state/probes/workbench.mjs",
"bun scripts/cli.ts hwlab nodes web-probe observe start --node D601 --lane v03 --target-path /workbench --sample-interval-ms 5000",
"bun scripts/cli.ts hwlab nodes web-probe observe command --node D601 --lane v03 --state-dir .state/web-observe/D601/v03/YYYY/MM/DD/<run> --type sendPrompt --text 'ping'",
"bun scripts/cli.ts hwlab nodes web-probe observe status --node D601 --lane v03 --state-dir .state/web-observe/D601/v03/YYYY/MM/DD/<run>",
"bun scripts/cli.ts hwlab nodes web-probe observe analyze --node D601 --lane v03 --state-dir .state/web-observe/D601/v03/YYYY/MM/DD/<run>",
"bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 <<'JS'\nexport default async ({ waitWorkbenchReady, fetchJson, fetchApiMatrix, recordStep, collectText, safeEvaluate, screenshot }) => {\n const ready = await waitWorkbenchReady();\n const workspace = await fetchJson('/v1/workbench/workspace?projectId=prj_hwpod_workbench');\n const apiMatrix = await fetchApiMatrix(['/v1/workbench/workspace?projectId=prj_hwpod_workbench', '/auth/session']);\n const workspaceText = await collectText('#workspace');\n const evaluated = await safeEvaluate(({ a, b }) => ({ sum: a + b }), { a: 1, b: 2 });\n await screenshot('workbench.png');\n recordStep('workbench-summary', { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, apiMatrixOk: apiMatrix.ok });\n return { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, workspaceText, evaluated };\n};\nJS",
],
actions: {
run: "Run the repo-owned scripts/web-live-dom-probe.mjs helper.",
script: "Run caller-provided Playwright JS after CLI-managed /auth/login; the script receives authenticated browser/context/page plus gotoStable/reloadStable/gotoCurrentStable/safeReload/fetchJson/safeFetchJson/fetchApiMatrix/recordStep/collectText/safeEvaluate/waitWorkbenchReady/screenshotOnError/summarizeWorkspace/summarizeConversation helpers and must not handle secrets itself.",
observe: "Start, inspect, control, stop, collect, and analyze a pure-client long-running Workbench observer on the target host. The observer has one Playwright page authority, receives commands through stateDir/commands files, writes JSONL artifacts, and does not expose any inbound service API.",
},
notes: [
"Prefer --script-file for reusable probes; stdin heredocs remain supported for one-off probes.",
"Issue-ready evidence is available under issueEvidence and summary.issueEvidence; full script report is persisted under probe.reportPath with a SHA-256 fingerprint.",
"observe sampling is passive by default: it records DOM summaries and natural page request/response/requestfailed events with observerInitiated=false; it does not actively fetch Workbench APIs, reload, switch sessions, route/intercept, or call repair helpers.",
"observe command actions are explicit user/control actions and are appended to control.jsonl; use --type sendPrompt/goto/screenshot/mark/stop and keep prompt text out of issue comments by citing textHash/textBytes.",
"observe analyze is offline-only: it reads artifact JSONL and writes analysis/report.md plus analysis/report.json without accessing Workbench APIs or driving the browser.",
"Use recordStep(name, data) or fetchApiMatrix(paths) to keep structured partial evidence when a later step fails.",
"Use reloadStable(), gotoCurrentStable(), or safeReload() for bounded retries around page reload/current-URL navigation jitter such as ERR_NETWORK_CHANGED.",
"Playwright page.evaluate accepts one serializable argument; use page.evaluate(({ a, b }) => ..., { a, b }) or safeEvaluate(fn, { a, b }).",
+424 -3
View File
@@ -10,6 +10,7 @@ import { classifySshTcpPoolFailure } from "./ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "./hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource, nodeWebObserveRunnerSource } from "./hwlab-node-web-observe-runner-source";
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "./hwlab-node-help";
import { compactWebProbeResult, compactWebProbeScriptResult } from "./hwlab-node-web-probe-summary";
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "./hwlab-node-observability-promql";
@@ -56,7 +57,35 @@ interface NodeWebProbeScriptOptions {
};
}
type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions;
type NodeWebProbeObserveAction = "start" | "status" | "command" | "stop" | "collect" | "analyze";
type NodeWebProbeObserveCommandType = "login" | "preflight" | "goto" | "sendPrompt" | "clickSession" | "screenshot" | "mark" | "stop";
interface NodeWebProbeObserveOptions {
action: "observe";
observeAction: NodeWebProbeObserveAction;
node: string;
lane: string;
url: string;
targetPath: string;
viewport: string;
sampleIntervalMs: number;
screenshotIntervalMs: number;
maxSamples: number;
commandTimeoutSeconds: number;
waitMs: number;
tailLines: number;
maxFiles: number;
stateDir: string | null;
jobId: string | null;
force: boolean;
commandType: NodeWebProbeObserveCommandType | null;
commandText: string | null;
commandPath: string | null;
commandLabel: string | null;
commandSessionId: string | null;
}
type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions | NodeWebProbeObserveOptions;
type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary" | "performance-summary";
@@ -5896,13 +5925,14 @@ function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof par
function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
const [actionRaw] = args;
if (actionRaw !== "run" && actionRaw !== "script") throw new Error("web-probe usage: run|script --node NODE --lane vNN [--url URL]");
if (actionRaw !== "run" && actionRaw !== "script" && actionRaw !== "observe") throw new Error("web-probe usage: run|script|observe --node NODE --lane vNN [--url URL]");
const node = requiredOption(args, "--node");
assertNodeId(node);
const lane = requiredOption(args, "--lane");
assertLane(lane);
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
const spec = hwlabRuntimeLaneSpecForNode(lane, node);
if (actionRaw === "observe") return parseNodeWebProbeObserveOptions(args.slice(1), node, lane, spec);
if (actionRaw === "script") {
assertKnownOptions(args.slice(1), new Set([
"--node",
@@ -6001,6 +6031,88 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
};
}
function parseNodeWebProbeObserveOptions(args: string[], node: string, lane: string, spec: HwlabRuntimeLaneSpec): NodeWebProbeObserveOptions {
const [observeActionRaw] = args;
if (
observeActionRaw !== "start"
&& observeActionRaw !== "status"
&& observeActionRaw !== "command"
&& observeActionRaw !== "stop"
&& observeActionRaw !== "collect"
&& observeActionRaw !== "analyze"
) {
throw new Error("web-probe observe usage: observe start|status|command|stop|collect|analyze --node NODE --lane vNN [--state-dir DIR|--job-id ID]");
}
assertKnownOptions(args.slice(1), new Set([
"--node",
"--lane",
"--url",
"--target-path",
"--viewport",
"--sample-interval-ms",
"--screenshot-interval-ms",
"--max-samples",
"--command-timeout-seconds",
"--wait-ms",
"--tail-lines",
"--max-files",
"--state-dir",
"--job-id",
"--type",
"--text",
"--path",
"--label",
"--session-id",
]), new Set(["--force"]));
const commandTypeRaw = optionValue(args, "--type") ?? null;
const commandType = commandTypeRaw === null ? null : parseNodeWebProbeObserveCommandType(commandTypeRaw);
const stateDir = optionValue(args, "--state-dir") ?? null;
const jobId = optionValue(args, "--job-id") ?? null;
if (stateDir !== null && !isSafeWebObserveStateDir(stateDir)) throw new Error(`unsafe web-probe observe --state-dir: ${stateDir}`);
if (jobId !== null && !isSafeWebObserveJobId(jobId)) throw new Error(`unsafe web-probe observe --job-id: ${jobId}`);
if (observeActionRaw !== "start" && stateDir === null && jobId === null) {
throw new Error("web-probe observe status|command|stop|collect|analyze requires --state-dir or --job-id");
}
return {
action: "observe",
observeAction: observeActionRaw,
node,
lane,
url: optionValue(args, "--url") ?? spec.publicWebUrl,
targetPath: optionValue(args, "--target-path") ?? "/workbench",
viewport: optionValue(args, "--viewport") ?? "1440x900",
sampleIntervalMs: positiveIntegerOption(args, "--sample-interval-ms", 5000, 600000),
screenshotIntervalMs: positiveIntegerOption(args, "--screenshot-interval-ms", 300000, 86_400_000),
maxSamples: positiveIntegerOption(args, "--max-samples", 0, 10_000_000),
commandTimeoutSeconds: positiveIntegerOption(args, "--command-timeout-seconds", 55, 3600),
waitMs: positiveIntegerOption(args, "--wait-ms", observeActionRaw === "command" || observeActionRaw === "stop" ? 30000 : 0, 600000),
tailLines: positiveIntegerOption(args, "--tail-lines", 5, 200),
maxFiles: positiveIntegerOption(args, "--max-files", 80, 5000),
stateDir,
jobId,
force: args.includes("--force"),
commandType,
commandText: optionValue(args, "--text") ?? null,
commandPath: optionValue(args, "--path") ?? null,
commandLabel: optionValue(args, "--label") ?? null,
commandSessionId: optionValue(args, "--session-id") ?? null,
};
}
function parseNodeWebProbeObserveCommandType(value: string): NodeWebProbeObserveCommandType {
if (
value === "login"
|| value === "preflight"
|| value === "goto"
|| value === "sendPrompt"
|| value === "clickSession"
|| value === "screenshot"
|| value === "mark"
|| value === "stop"
) return value;
throw new Error(`web-probe observe command --type must be login, preflight, goto, sendPrompt, clickSession, screenshot, mark, or stop; got ${value}`);
}
function nodeWebProbeAutoCommandTimeoutSeconds(input: {
timeoutMs: number;
waitAfterSubmitMs: number;
@@ -6039,6 +6151,7 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
const lane = options.lane;
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
const spec = hwlabRuntimeLaneSpecForNode(lane, options.node);
if (options.action === "observe" && options.observeAction !== "start") return runNodeWebProbeObserve(options, spec, null, null, null);
const secretSpec = runtimeSecretSpec({ node: options.node, lane });
const material = readBootstrapAdminPasswordMaterial(secretSpec);
const credential = webProbeCredential(secretSpec, material);
@@ -6046,7 +6159,9 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
return {
ok: false,
status: "blocked",
command: `hwlab nodes web-probe ${options.action} --node ${options.node} --lane ${options.lane}`,
command: options.action === "observe"
? `hwlab nodes web-probe observe ${options.observeAction} --node ${options.node} --lane ${options.lane}`
: `hwlab nodes web-probe ${options.action} --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
workspace: spec.workspace,
@@ -6056,6 +6171,7 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
next: { secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${secretSpec.bootstrapAdminSecret}` },
};
}
if (options.action === "observe") return runNodeWebProbeObserve(options, spec, secretSpec, material, credential);
if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential);
if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential);
const probeArgs = nodeWebProbeRunArgs(options, "run");
@@ -6279,6 +6395,311 @@ function webProbeCredential(secretSpec: RuntimeSecretSpec, material: BootstrapAd
};
}
function runNodeWebProbeObserve(
options: NodeWebProbeObserveOptions,
spec: HwlabRuntimeLaneSpec,
secretSpec: RuntimeSecretSpec | null,
material: BootstrapAdminPasswordMaterial | null,
credential: Record<string, unknown> | null,
): Record<string, unknown> {
if (options.observeAction === "start") {
if (secretSpec === null || material === null || credential === null || material.password === null) throw new Error("web-probe observe start requires bootstrap admin credential material");
return runNodeWebProbeObserveStart(options, spec, secretSpec, material, credential);
}
if (options.observeAction === "status") return runNodeWebProbeObserveStatus(options, spec);
if (options.observeAction === "command") return runNodeWebProbeObserveCommand(options, spec, false);
if (options.observeAction === "stop") return runNodeWebProbeObserveCommand({ ...options, commandType: "stop" }, spec, true);
if (options.observeAction === "collect") return runNodeWebProbeObserveCollect(options, spec);
return runNodeWebProbeObserveAnalyze(options, spec);
}
function runNodeWebProbeObserveStart(
options: NodeWebProbeObserveOptions,
spec: HwlabRuntimeLaneSpec,
secretSpec: RuntimeSecretSpec,
material: BootstrapAdminPasswordMaterial,
credential: Record<string, unknown>,
): Record<string, unknown> {
const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
const day = timestamp.slice(0, 8);
const stateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64");
const script = [
"set -eu",
`state_dir=${shellQuote(stateDir)}`,
"mkdir -p \"$state_dir\"",
"chmod 700 \"$state_dir\"",
"runner=\"$state_dir/observer-runner.mjs\"",
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
"chmod 700 \"$runner\"",
[
`HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
`HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`,
`HWLAB_WEB_PASS=${shellQuote(material.password)}`,
`UNIDESK_WEB_OBSERVE_STATE_DIR=${shellQuote(stateDir)}`,
`UNIDESK_WEB_OBSERVE_JOB_ID=${shellQuote(jobId)}`,
`UNIDESK_WEB_OBSERVE_TARGET_PATH=${shellQuote(options.targetPath)}`,
`UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS=${shellQuote(String(options.sampleIntervalMs))}`,
`UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS=${shellQuote(String(options.screenshotIntervalMs))}`,
`UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`,
`UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`,
"nohup node \"$runner\" >\"$state_dir/stdout.log\" 2>\"$state_dir/stderr.log\" </dev/null &",
].join(" "),
"pid=$!",
"printf '%s\\n' \"$pid\" >\"$state_dir/pid\"",
"sleep 1",
`node -e ${shellQuote("const fs=require('fs'); const dir=process.argv[1]; const read=(n)=>{try{return JSON.parse(fs.readFileSync(dir+'/'+n,'utf8'))}catch{return null}}; const pid=fs.existsSync(dir+'/pid')?fs.readFileSync(dir+'/pid','utf8').trim():null; console.log(JSON.stringify({ok:true,command:'web-probe-observe start',jobId:process.argv[2],stateDir:dir,pid:Number(pid)||null,manifestPath:dir+'/manifest.json',heartbeat:read('heartbeat.json'),manifest:read('manifest.json'),statusCommand:'bun scripts/cli.ts hwlab nodes web-probe observe status --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,stopCommand:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+process.argv[3]+' --lane '+process.argv[4]+' --state-dir '+dir,valuesRedacted:true},null,2))")} "$state_dir" ${shellQuote(jobId)} ${shellQuote(options.node)} ${shellQuote(options.lane)}`,
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const started = parseJsonObject(result.stdout);
return {
ok: result.exitCode === 0 && started?.ok === true,
status: result.exitCode === 0 && started?.ok === true ? "started" : "blocked",
command: `hwlab nodes web-probe observe start --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
workspace: spec.workspace,
url: options.url,
targetPath: options.targetPath,
credential,
observer: started,
result: compactCommandResultRedacted(result, [material.password ?? ""]),
valuesRedacted: true,
};
}
function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const script = [
"set -eu",
nodeWebObserveResolveStateDirShell(options),
nodeWebObserveStatusNodeScript(options.tailLines, options.node, options.lane),
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const status = parseJsonObject(result.stdout);
return {
ok: result.exitCode === 0 && status?.ok !== false,
status: result.exitCode === 0 ? "observed" : "blocked",
command: `hwlab nodes web-probe observe status --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
workspace: spec.workspace,
observer: status,
result: compactCommandResult(result),
valuesRedacted: true,
};
}
function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record<string, unknown> {
const type = options.commandType ?? (stopCommand ? "stop" : null);
if (type === null) throw new Error("web-probe observe command requires --type");
const commandId = `cmd-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
const payload = {
id: commandId,
type,
createdAt: new Date().toISOString(),
source: "cli",
path: options.commandPath,
text: options.commandText,
label: options.commandLabel,
sessionId: options.commandSessionId,
};
const payloadB64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
const script = [
"set -eu",
nodeWebObserveResolveStateDirShell(options),
"mkdir -p \"$state_dir/commands/pending\"",
`node -e "const fs=require('fs'),path=require('path'); const dir=process.argv[1], id=process.argv[2], payload=Buffer.from(process.argv[3], 'base64').toString('utf8'); fs.writeFileSync(path.join(dir,'commands','pending',id+'.json'), payload+'\\n', {mode:0o600});" "$state_dir" ${shellQuote(commandId)} ${shellQuote(payloadB64)}`,
nodeWebObserveWaitCommandShell(commandId, options.waitMs),
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const commandResult = parseJsonObject(result.stdout);
if (options.force && stopCommand && result.exitCode !== 0) {
const killResult = runTransWorkspaceStdinScript(options.node, spec.workspace, [
"set -eu",
nodeWebObserveResolveStateDirShell(options),
"if [ -f \"$state_dir/pid\" ]; then kill \"$(cat \"$state_dir/pid\")\" >/dev/null 2>&1 || true; fi",
"printf '{\"ok\":true,\"forced\":true,\"stateDir\":\"%s\"}\\n' \"$state_dir\"",
].join("\n"), 55);
return {
ok: killResult.exitCode === 0,
status: killResult.exitCode === 0 ? "forced-stop-requested" : "blocked",
command: `hwlab nodes web-probe observe stop --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
workspace: spec.workspace,
commandId,
observerCommand: commandSummaryForOutput(payload),
gracefulResult: compactCommandResult(result),
forceResult: compactCommandResult(killResult),
valuesRedacted: true,
};
}
return {
ok: result.exitCode === 0 && commandResult?.ok !== false,
status: result.exitCode === 0 ? (options.waitMs > 0 ? "completed-or-queued" : "queued") : "blocked",
command: stopCommand
? `hwlab nodes web-probe observe stop --node ${options.node} --lane ${options.lane}`
: `hwlab nodes web-probe observe command --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
workspace: spec.workspace,
commandId,
observerCommand: commandSummaryForOutput(payload),
observer: commandResult,
result: compactCommandResult(result),
valuesRedacted: true,
};
}
function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const script = [
"set -eu",
nodeWebObserveResolveStateDirShell(options),
nodeWebObserveCollectNodeScript(options.maxFiles),
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const collect = parseJsonObject(result.stdout);
return {
ok: result.exitCode === 0 && collect?.ok !== false,
status: result.exitCode === 0 ? "collected" : "blocked",
command: `hwlab nodes web-probe observe collect --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
workspace: spec.workspace,
collect,
result: compactCommandResult(result),
valuesRedacted: true,
};
}
function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const analyzerB64 = Buffer.from(nodeWebObserveAnalyzerSource(), "utf8").toString("base64");
const script = [
"set -eu",
nodeWebObserveResolveStateDirShell(options),
"analyzer=\"$state_dir/observer-analyzer.mjs\"",
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$analyzer" ${shellQuote(analyzerB64)}`,
"chmod 700 \"$analyzer\"",
"UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" node \"$analyzer\"",
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const analysis = parseJsonObject(result.stdout);
return {
ok: result.exitCode === 0 && analysis?.ok === true,
status: result.exitCode === 0 && analysis?.ok === true ? "analyzed" : "blocked",
command: `hwlab nodes web-probe observe analyze --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
workspace: spec.workspace,
analysis,
result: compactCommandResult(result),
valuesRedacted: true,
};
}
function nodeWebObserveResolveStateDirShell(options: Pick<NodeWebProbeObserveOptions, "stateDir" | "jobId" | "node" | "lane">): string {
if (options.stateDir !== null) {
return [
`state_dir=${shellQuote(options.stateDir)}`,
"test -d \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"state-dir-missing\",\"stateDir\":\"%s\"}\\n' \"$state_dir\"; exit 2; }",
].join("\n");
}
if (options.jobId === null) throw new Error("web-probe observe requires --state-dir or --job-id");
const root = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}`;
return [
`observe_root=${shellQuote(root)}`,
`job_id=${shellQuote(options.jobId)}`,
"state_dir=$(find \"$observe_root\" -type d -name \"*_${job_id}\" 2>/dev/null | sort | tail -n 1 || true)",
"test -n \"$state_dir\" || { printf '{\"ok\":false,\"error\":\"job-id-not-found\",\"jobId\":\"%s\",\"root\":\"%s\"}\\n' \"$job_id\" \"$observe_root\"; exit 2; }",
].join("\n");
}
function nodeWebObserveStatusNodeScript(tailLines: number, node: string, lane: string): string {
return `node -e ${shellQuote(`
const fs=require('fs'),path=require('path');
const dir=process.env.state_dir||process.argv[1];
const node=process.argv[2];
const lane=process.argv[3];
const tailN=${tailLines};
const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name),'utf8'))}catch{return null}};
const tailJsonl=(name)=>{try{return fs.readFileSync(path.join(dir,name),'utf8').trim().split(/\\r?\\n/).filter(Boolean).slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}})}catch{return []}};
const short=(value)=>String(value||'').slice(0,160);
const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt}:null;
const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,source:item.source,input:item.input,afterUrl:item.afterUrl,detail:item.detail&&typeof item.detail==='object'?{reason:item.detail.reason||null,mark:item.detail.mark||null,stopping:item.detail.stopping===true,pageId:item.detail.pageId||null}:null});
const compactSample=(item)=>({seq:item.seq,ts:item.ts,reason:item.reason,url:item.url,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,observerInitiated:item.observerInitiated===true,scroll:item.scroll||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0,messages:Array.isArray(item.messages)?item.messages.slice(-3).map((row)=>({index:row.index,tag:row.tag,status:row.status||null,textHash:row.textHash||null,textPreview:short(row.textPreview),textBytes:row.textBytes||0})):[],traceRows:Array.isArray(item.traceRows)?item.traceRows.slice(-3).map((row)=>({index:row.index,tag:row.tag,status:row.status||null,textHash:row.textHash||null,textPreview:short(row.textPreview),textBytes:row.textBytes||0})):[],performanceCount:Array.isArray(item.performance)?item.performance.length:0});
const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:item.url,resourceType:item.resourceType,status:item.status||null,observerInitiated:item.observerInitiated===true,bodyRead:item.bodyRead===true,commandId:item.commandId||null,failure:item.failure||null});
const pidText=fs.existsSync(path.join(dir,'pid'))?fs.readFileSync(path.join(dir,'pid'),'utf8').trim():null;
let alive=false; if(pidText){try{process.kill(Number(pidText),0); alive=true}catch{}}
console.log(JSON.stringify({ok:true,command:'web-probe-observe status',stateDir:dir,pid:pidText?Number(pidText):null,processAlive:alive,manifest:compactManifest(readJson('manifest.json')),heartbeat:readJson('heartbeat.json'),tails:{control:tailJsonl('control.jsonl').map(compactControl),samples:tailJsonl('samples.jsonl').map(compactSample),network:tailJsonl('network.jsonl').map(compactNetwork),errors:tailJsonl('errors.jsonl'),artifacts:tailJsonl('artifacts.jsonl')},next:{command:'bun scripts/cli.ts hwlab nodes web-probe observe command --node '+node+' --lane '+lane+' --state-dir '+dir+' --type mark --label checkpoint',stop:'bun scripts/cli.ts hwlab nodes web-probe observe stop --node '+node+' --lane '+lane+' --state-dir '+dir,analyze:'bun scripts/cli.ts hwlab nodes web-probe observe analyze --node '+node+' --lane '+lane+' --state-dir '+dir},valuesRedacted:true},null,2));
`)} "$state_dir" ${shellQuote(node)} ${shellQuote(lane)}`;
}
function nodeWebObserveCollectNodeScript(maxFiles: number): string {
return `node -e ${shellQuote(`
const fs=require('fs'),path=require('path'),crypto=require('crypto');
const dir=process.argv[1]; const maxFiles=${maxFiles};
const out=[]; const walk=(p)=>{for(const ent of fs.readdirSync(p,{withFileTypes:true})){const full=path.join(p,ent.name); if(ent.isDirectory()) walk(full); else out.push(full); if(out.length>=maxFiles) return;}};
walk(dir);
const files=out.slice(0,maxFiles).map(file=>{const st=fs.statSync(file); const hash=crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); return {path:file,relative:path.relative(dir,file),byteCount:st.size,sha256:'sha256:'+hash}});
const totalBytes=files.reduce((sum,item)=>sum+item.byteCount,0);
console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',stateDir:dir,fileCount:files.length,totalBytes,files,valuesRedacted:true},null,2));
`)} "$state_dir"`;
}
function nodeWebObserveWaitCommandShell(commandId: string, waitMs: number): string {
if (waitMs <= 0) {
return [
`printf '{"ok":true,"queued":true,"commandId":%s,"stateDir":%s}\\n' ${shellQuote(JSON.stringify(commandId))} "$(node -e "console.log(JSON.stringify(process.argv[1]))" "$state_dir")"`,
].join("\n");
}
const waitSeconds = Math.max(1, Math.ceil(waitMs / 1000));
return [
`command_id=${shellQuote(commandId)}`,
`deadline=$(( $(date +%s) + ${waitSeconds} ))`,
"while [ \"$(date +%s)\" -le \"$deadline\" ]; do",
" if [ -f \"$state_dir/commands/done/${command_id}.json\" ]; then cat \"$state_dir/commands/done/${command_id}.json\"; exit 0; fi",
" if [ -f \"$state_dir/commands/failed/${command_id}.json\" ]; then cat \"$state_dir/commands/failed/${command_id}.json\"; exit 2; fi",
" sleep 1",
"done",
"printf '{\"ok\":true,\"queued\":true,\"waitTimedOut\":true,\"commandId\":\"%s\",\"stateDir\":\"%s\"}\\n' \"$command_id\" \"$state_dir\"",
].join("\n");
}
function commandSummaryForOutput(payload: Record<string, unknown>): Record<string, unknown> {
const text = typeof payload.text === "string" ? payload.text : null;
return {
id: payload.id,
type: payload.type,
path: payload.path ?? null,
label: payload.label ?? null,
sessionId: payload.sessionId ?? null,
textHash: text === null ? null : `sha256:${createHash("sha256").update(text).digest("hex")}`,
textBytes: text === null ? null : Buffer.byteLength(text),
valuesRedacted: true,
};
}
function isSafeWebObserveStateDir(value: string): boolean {
return value.length > 0
&& !value.includes("\0")
&& !value.includes("..")
&& /^\.state\/web-observe\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/\d{4}\/\d{2}\/\d{2}\/[A-Za-z0-9_.TZ-]+_[A-Za-z0-9_.-]+_webobs-[A-Za-z0-9_.-]+$/u.test(value);
}
function isSafeWebObserveJobId(value: string): boolean {
return /^webobs-[A-Za-z0-9_.-]+$/u.test(value);
}
function safeWebObserveSegment(value: string): string {
return value.replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "") || "item";
}
function safeWebObserveTargetSegment(value: string): string {
const segment = value.replace(/^https?:\/\//u, "").replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "");
return segment.slice(0, 48) || "workbench";
}
function runNodeWebProbeScript(
options: NodeWebProbeScriptOptions,
spec: HwlabRuntimeLaneSpec,
@@ -0,0 +1,716 @@
// SPEC: PJ2026-01040111 长程观测 draft-2026-06-20-p0-passive-web-probe-observer.
// Responsibility: Source strings for the pure-client HWLAB web-probe observer and offline analyzer.
export function nodeWebObserveRunnerSource(): string {
return String.raw`#!/usr/bin/env node
import { createHash, randomBytes } from "node:crypto";
import { appendFile, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
const specRef = "PJ2026-01040111 长程观测 draft-2026-06-20-p0-passive-web-probe-observer";
const startedAtMs = Date.now();
const startedAt = new Date(startedAtMs).toISOString();
const baseUrl = normalizeBaseUrl(process.env.HWLAB_WEB_BASE_URL);
const username = process.env.HWLAB_WEB_USER || "admin";
const password = process.env.HWLAB_WEB_PASS || "";
const stateDir = path.resolve(process.env.UNIDESK_WEB_OBSERVE_STATE_DIR || ".state/web-observe/manual");
const jobId = safeId(process.env.UNIDESK_WEB_OBSERVE_JOB_ID || "webobs-" + Date.now().toString(36) + "-" + randomBytes(3).toString("hex"));
const targetPath = process.env.UNIDESK_WEB_OBSERVE_TARGET_PATH || "/workbench";
const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_INTERVAL_MS, 5000);
const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000);
const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0);
const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900");
const pageId = "page-" + randomBytes(4).toString("hex");
const dirs = {
commandsPending: path.join(stateDir, "commands", "pending"),
commandsProcessing: path.join(stateDir, "commands", "processing"),
commandsDone: path.join(stateDir, "commands", "done"),
commandsFailed: path.join(stateDir, "commands", "failed"),
screenshots: path.join(stateDir, "screenshots"),
analysis: path.join(stateDir, "analysis"),
};
const files = {
manifest: path.join(stateDir, "manifest.json"),
heartbeat: path.join(stateDir, "heartbeat.json"),
control: path.join(stateDir, "control.jsonl"),
samples: path.join(stateDir, "samples.jsonl"),
network: path.join(stateDir, "network.jsonl"),
console: path.join(stateDir, "console.jsonl"),
errors: path.join(stateDir, "errors.jsonl"),
artifacts: path.join(stateDir, "artifacts.jsonl"),
};
let browser;
let context;
let page;
let sampleSeq = 0;
let commandSeq = 0;
let artifactSeq = 0;
let activeCommandId = null;
let stopping = false;
let terminalStatus = "starting";
let lastScreenshotAtMs = 0;
let auth = null;
try {
if (!password) throw new Error("missing HWLAB_WEB_PASS");
await prepareDirs();
await writeManifest({ status: "starting" });
await writeHeartbeat({ status: "starting" });
const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
const { chromium } = await launcher.importPlaywright();
browser = await launcher.launchChromium(chromium);
context = await browser.newContext({ viewport });
auth = await runControlCommand({ id: "startup-login", type: "login", createdAt: startedAt, source: "startup" }, async () => authenticate(context));
page = await context.newPage();
attachPassiveListeners(page);
await runControlCommand({ id: "startup-goto", type: "goto", path: targetPath, createdAt: new Date().toISOString(), source: "startup" }, async () => gotoTarget(targetPath));
terminalStatus = "running";
await writeManifest({ status: "running", auth: publicAuth(auth) });
await writeHeartbeat({ status: "running" });
while (!stopping) {
await drainOneCommand();
await samplePage("interval");
if (maxSamples > 0 && sampleSeq >= maxSamples) {
await appendJsonl(files.control, controlRecord({ id: "max-samples", type: "stop", source: "sampler" }, "completed", { reason: "max-samples", maxSamples }));
break;
}
await sleep(sampleIntervalMs);
}
terminalStatus = "completed";
await writeHeartbeat({ status: "completed" });
await writeManifest({ status: "completed", completedAt: new Date().toISOString() });
process.exitCode = 0;
} catch (error) {
terminalStatus = "failed";
await appendJsonl(files.errors, eventRecord("runner-error", { error: errorSummary(error) })).catch(() => {});
await writeHeartbeat({ status: "failed", error: errorSummary(error) }).catch(() => {});
await writeManifest({ status: "failed", error: errorSummary(error) }).catch(() => {});
process.exitCode = 2;
} finally {
if (browser) await browser.close().catch(() => {});
}
async function prepareDirs() {
await mkdir(stateDir, { recursive: true, mode: 0o700 });
await Promise.all(Object.values(dirs).map((dir) => mkdir(dir, { recursive: true, mode: 0o700 })));
}
async function writeManifest(extra = {}) {
const manifest = {
ok: extra.status !== "failed",
command: "web-probe-observe",
specRef,
jobId,
pid: process.pid,
stateDir,
baseUrl,
targetPath,
pageAuthority: { browser: "chromium", context: "single", pageId, continuityBreaksRecorded: true },
sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerInitiatedDefault: false, responseBodyReadDefault: false },
commandDirs: dirs,
artifacts: files,
safety: { pureClient: true, inboundApi: false, database: false, queueConsumer: false, k8sWorkload: false, valuesRedacted: true, secretValuesPrinted: false },
startedAt,
...extra,
};
await writeFile(files.manifest, JSON.stringify(manifest, null, 2) + "\n", { mode: 0o600 });
}
async function writeHeartbeat(extra = {}) {
const heartbeat = {
ok: terminalStatus !== "failed",
jobId,
pid: process.pid,
stateDir,
status: terminalStatus,
pageId,
baseUrl,
currentUrl: currentPageUrl(),
sampleSeq,
commandSeq,
activeCommandId,
updatedAt: new Date().toISOString(),
uptimeMs: Date.now() - startedAtMs,
...extra,
};
await writeFile(files.heartbeat, JSON.stringify(heartbeat, null, 2) + "\n", { mode: 0o600 });
}
function attachPassiveListeners(targetPage) {
targetPage.on("request", (request) => {
void appendJsonl(files.network, eventRecord("request", {
observerInitiated: false,
commandId: activeCommandId,
method: request.method(),
url: safeUrl(request.url()),
resourceType: request.resourceType(),
frameUrl: safeFrameUrl(request.frame()),
}));
});
targetPage.on("response", (response) => {
const request = response.request();
void appendJsonl(files.network, eventRecord("response", {
observerInitiated: false,
commandId: activeCommandId,
method: request.method(),
url: safeUrl(response.url()),
resourceType: request.resourceType(),
status: response.status(),
statusText: response.statusText(),
fromServiceWorker: response.fromServiceWorker(),
bodyRead: false,
}));
});
targetPage.on("requestfailed", (request) => {
void appendJsonl(files.network, eventRecord("requestfailed", {
observerInitiated: false,
commandId: activeCommandId,
method: request.method(),
url: safeUrl(request.url()),
resourceType: request.resourceType(),
failure: request.failure()?.errorText ?? null,
}));
});
targetPage.on("console", (message) => {
void appendJsonl(files.console, eventRecord("console", { type: message.type(), text: truncate(message.text(), 1000), location: message.location() }));
});
targetPage.on("pageerror", (error) => {
void appendJsonl(files.errors, eventRecord("pageerror", { error: errorSummary(error) }));
});
targetPage.on("crash", () => {
void appendJsonl(files.errors, eventRecord("page-crash", { pageId }));
});
targetPage.on("close", () => {
void appendJsonl(files.control, eventRecord("continuity-break", { pageId, reason: "page-closed" }));
});
}
async function drainOneCommand() {
const entries = (await readdir(dirs.commandsPending).catch(() => [])).filter((name) => name.endsWith(".json")).sort();
const name = entries[0];
if (!name) return;
const pending = path.join(dirs.commandsPending, name);
const processing = path.join(dirs.commandsProcessing, name);
await rename(pending, processing).catch(() => null);
const raw = await readFile(processing, "utf8");
const command = JSON.parse(raw);
const id = safeId(command.id || name.replace(/[.]json$/u, ""));
command.id = id;
try {
const result = await processCommand(command);
const done = { ok: true, commandId: id, type: command.type, completedAt: new Date().toISOString(), result: sanitize(result) };
await writeFile(path.join(dirs.commandsDone, id + ".json"), JSON.stringify(done, null, 2) + "\n", { mode: 0o600 });
await appendJsonl(files.control, controlRecord(command, "completed", done.result));
await unlink(processing).catch(() => {});
} catch (error) {
const failed = { ok: false, commandId: id, type: command.type, failedAt: new Date().toISOString(), error: errorSummary(error) };
await writeFile(path.join(dirs.commandsFailed, id + ".json"), JSON.stringify(failed, null, 2) + "\n", { mode: 0o600 });
await appendJsonl(files.control, controlRecord(command, "failed", failed.error));
await unlink(processing).catch(() => {});
} finally {
activeCommandId = null;
await writeHeartbeat({ status: terminalStatus });
}
}
async function processCommand(command) {
commandSeq += 1;
activeCommandId = command.id;
await writeHeartbeat({ status: "running", activeCommandId });
await appendJsonl(files.control, controlRecord(command, "started", commandInputSummary(command)));
switch (command.type) {
case "login": return authenticate(context);
case "preflight": return preflightSummary();
case "goto": return gotoTarget(command.path || command.url || targetPath);
case "sendPrompt": return sendPrompt(String(command.text || ""));
case "clickSession": return clickSession(String(command.sessionId || command.value || ""));
case "screenshot": return captureScreenshot(command.reason || "manual", command.imageType || "png");
case "mark": return { mark: truncate(command.label || command.text || "mark", 200), currentUrl: currentPageUrl(), pageId };
case "stop": stopping = true; return { stopping: true, currentUrl: currentPageUrl(), pageId };
default: throw new Error("unsupported observer command type: " + command.type);
}
}
async function runControlCommand(command, fn) {
activeCommandId = command.id;
commandSeq += 1;
const beforeUrl = currentPageUrl();
const started = Date.now();
await appendJsonl(files.control, controlRecord(command, "started", { beforeUrl, input: commandInputSummary(command) }));
try {
const result = await fn();
await appendJsonl(files.control, controlRecord(command, "completed", { beforeUrl, afterUrl: currentPageUrl(), durationMs: Date.now() - started, result: sanitize(result) }));
return result;
} catch (error) {
await appendJsonl(files.control, controlRecord(command, "failed", { beforeUrl, afterUrl: currentPageUrl(), durationMs: Date.now() - started, error: errorSummary(error) }));
throw error;
} finally {
activeCommandId = null;
}
}
async function authenticate(browserContext) {
const loginUrl = new URL("/auth/login", baseUrl).toString();
const response = await browserContext.request.post(loginUrl, {
data: { username, password },
headers: { accept: "application/json", "content-type": "application/json" },
timeout: 15000,
});
const cookies = await browserContext.cookies(baseUrl);
const cookieNames = cookies.map((cookie) => cookie.name).filter((name) => /session|auth|token/iu.test(name));
return {
ok: response.ok() && cookieNames.length > 0,
method: "api",
loginPath: new URL(loginUrl).pathname,
status: response.status(),
statusText: response.statusText(),
cookiePresent: cookieNames.length > 0,
cookieNames,
valuesRedacted: true,
};
}
function publicAuth(value) {
if (!value) return null;
return { ok: value.ok === true, method: value.method, status: value.status, cookiePresent: value.cookiePresent === true, cookieNames: value.cookieNames || [], valuesRedacted: true };
}
async function gotoTarget(rawTarget) {
const target = new URL(String(rawTarget || targetPath), baseUrl).toString();
const beforeUrl = currentPageUrl();
const response = await page.goto(target, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(1000).catch(() => {});
return { beforeUrl, afterUrl: currentPageUrl(), httpStatus: response ? response.status() : null, pageId };
}
async function sendPrompt(text) {
if (text.trim().length === 0) throw new Error("sendPrompt requires non-empty text");
const beforeUrl = currentPageUrl();
const editor = page.locator('textarea, [role="textbox"], [contenteditable="true"], input[type="text"]').last();
await editor.waitFor({ state: "visible", timeout: 15000 });
const tag = await editor.evaluate((element) => element.tagName.toLowerCase()).catch(() => "");
const editable = await editor.evaluate((element) => element.getAttribute("contenteditable") === "true").catch(() => false);
if (tag === "textarea" || tag === "input") await editor.fill(text);
else if (editable) {
await editor.click();
await page.keyboard.insertText(text);
} else {
await editor.click();
await page.keyboard.insertText(text);
}
const submit = page.locator([
'button[type="submit"]',
'button:has-text("发送")',
'button:has-text("Send")',
'[data-testid*="send" i]',
'[aria-label*="send" i]',
'[aria-label*="发送"]'
].join(", ")).last();
await submit.waitFor({ state: "visible", timeout: 15000 });
await submit.click();
await page.waitForTimeout(1500);
return { beforeUrl, afterUrl: currentPageUrl(), textHash: sha256Text(text), textBytes: Buffer.byteLength(text), pageId };
}
async function clickSession(sessionId) {
if (!sessionId) throw new Error("clickSession requires --session-id or --value");
const beforeUrl = currentPageUrl();
const escaped = cssEscape(sessionId);
const candidate = page.locator("[data-session-id=\"" + escaped + "\"], [href*=\"" + escaped + "\"], text=" + sessionId).first();
await candidate.waitFor({ state: "visible", timeout: 15000 });
await candidate.click();
await page.waitForTimeout(1000);
return { beforeUrl, afterUrl: currentPageUrl(), sessionId, pageId };
}
async function preflightSummary() {
return { currentUrl: currentPageUrl(), title: await page.title().catch(() => null), pageId, auth: publicAuth(auth) };
}
async function samplePage(reason) {
if (!page || page.isClosed()) return;
sampleSeq += 1;
const dom = await page.evaluate(() => {
const trim = (value, limit = 500) => String(value || "").replace(/\s+/g, " ").trim().slice(0, limit);
const visible = (element) => {
const rect = element.getBoundingClientRect();
const style = window.getComputedStyle(element);
return rect.width > 0 && rect.height > 0 && style.visibility !== "hidden" && style.display !== "none";
};
const textHashInput = (element) => trim(element.textContent || "", 800);
const summarize = (selector, limit) => Array.from(document.querySelectorAll(selector)).filter(visible).slice(-limit).map((element, index) => {
const rect = element.getBoundingClientRect();
return {
index,
tag: element.tagName.toLowerCase(),
testId: element.getAttribute("data-testid"),
role: element.getAttribute("role"),
status: element.getAttribute("data-status") || element.getAttribute("aria-busy") || null,
text: textHashInput(element),
rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
};
});
const url = location.href;
const routeSessionMatch = url.match(/\/workbench\/sessions\/([^/?#]+)/u);
const activeSession = document.querySelector('[data-active="true"][data-session-id], [aria-selected="true"][data-session-id], .active[data-session-id]');
const activeSessionId = activeSession ? activeSession.getAttribute("data-session-id") : null;
const messageSelector = '[data-testid*="message" i], [class*="message" i], article, [role="article"]';
const traceSelector = '[data-testid*="trace" i], [class*="trace" i], [data-trace-id], [data-testid*="event" i]';
const messages = summarize(messageSelector, 20);
const traceRows = summarize(traceSelector, 30);
const active = document.activeElement;
return {
url,
path: location.pathname,
routeSessionId: routeSessionMatch ? decodeURIComponent(routeSessionMatch[1]) : null,
activeSessionId,
title: document.title,
focus: active ? { tag: active.tagName.toLowerCase(), testId: active.getAttribute("data-testid"), role: active.getAttribute("role") } : null,
viewport: { width: window.innerWidth, height: window.innerHeight, devicePixelRatio: window.devicePixelRatio },
scroll: { x: Math.round(window.scrollX), y: Math.round(window.scrollY), height: Math.round(document.documentElement.scrollHeight), width: Math.round(document.documentElement.scrollWidth) },
messages,
traceRows,
performance: performance.getEntriesByType("resource").slice(-30).map((entry) => ({ name: entry.name.split(/[?#]/u)[0].slice(0, 240), initiatorType: entry.initiatorType, startTime: Math.round(entry.startTime), duration: Math.round(entry.duration) })),
};
}).catch((error) => ({ error: errorSummary(error), url: currentPageUrl() }));
const sample = {
seq: sampleSeq,
ts: new Date().toISOString(),
reason,
pageId,
commandId: activeCommandId,
observerInitiated: false,
...digestDom(dom),
};
await appendJsonl(files.samples, sample);
if (screenshotIntervalMs > 0 && Date.now() - lastScreenshotAtMs >= screenshotIntervalMs) {
await captureScreenshot("checkpoint", "jpeg").catch((error) => appendJsonl(files.errors, eventRecord("screenshot-error", { error: errorSummary(error) })));
}
await writeHeartbeat({ status: terminalStatus });
}
function digestDom(dom) {
if (dom && dom.error) return dom;
const messages = Array.isArray(dom.messages) ? dom.messages.map((item) => ({ ...item, textHash: sha256Text(item.text || ""), textPreview: truncate(item.text || "", 160), textBytes: Buffer.byteLength(item.text || "") })) : [];
const traceRows = Array.isArray(dom.traceRows) ? dom.traceRows.map((item) => ({ ...item, textHash: sha256Text(item.text || ""), textPreview: truncate(item.text || "", 160), textBytes: Buffer.byteLength(item.text || "") })) : [];
return { ...dom, messages, traceRows };
}
async function captureScreenshot(reason, imageType = "png") {
if (!page || page.isClosed()) throw new Error("page is not available for screenshot");
artifactSeq += 1;
const safeReason = safeId(String(reason || "manual")).slice(0, 40) || "manual";
const type = imageType === "jpeg" || imageType === "jpg" ? "jpeg" : "png";
const ext = type === "jpeg" ? "jpg" : "png";
const file = path.join(dirs.screenshots, String(sampleSeq).padStart(6, "0") + "_" + String(artifactSeq).padStart(4, "0") + "_" + safeReason + "." + ext);
const options = type === "jpeg" ? { path: file, type: "jpeg", quality: 70, fullPage: false } : { path: file, type: "png", fullPage: false };
await page.screenshot(options);
const meta = await fileMeta(file);
const artifact = { seq: artifactSeq, sampleSeq, ts: new Date().toISOString(), kind: "screenshot", reason, path: file, type, byteCount: meta.byteCount, sha256: meta.sha256, pageId, currentUrl: currentPageUrl() };
await appendJsonl(files.artifacts, artifact);
lastScreenshotAtMs = Date.now();
return artifact;
}
function eventRecord(type, data) {
return { ts: new Date().toISOString(), type, jobId, pageId, sampleSeq, commandId: activeCommandId, ...sanitize(data) };
}
function controlRecord(command, phase, detail) {
return {
ts: new Date().toISOString(),
seq: commandSeq,
phase,
commandId: command.id,
type: command.type,
source: command.source || "file",
input: commandInputSummary(command),
beforeUrl: command.beforeUrl || null,
afterUrl: currentPageUrl(),
pageId,
detail: sanitize(detail),
};
}
function commandInputSummary(command) {
const text = typeof command.text === "string" ? command.text : null;
return {
type: command.type,
path: command.path || null,
url: command.url ? safeUrl(command.url) : null,
sessionId: command.sessionId || command.value || null,
label: command.label ? truncate(command.label, 200) : null,
textHash: text === null ? null : sha256Text(text),
textBytes: text === null ? null : Buffer.byteLength(text),
textPreview: null,
valuesRedacted: true,
};
}
async function appendJsonl(file, value) {
await appendFile(file, JSON.stringify(sanitize(value)) + "\n", { mode: 0o600 });
}
async function fileMeta(file) {
const [buffer, stats] = await Promise.all([readFile(file), stat(file)]);
return { byteCount: stats.size, sha256: "sha256:" + createHash("sha256").update(buffer).digest("hex") };
}
function currentPageUrl() {
try { return page && !page.isClosed() ? page.url() : null; } catch { return null; }
}
function safeFrameUrl(frame) {
try { return frame ? safeUrl(frame.url()) : null; } catch { return null; }
}
function safeUrl(value) {
try {
const url = new URL(String(value), baseUrl);
for (const key of Array.from(url.searchParams.keys())) {
if (/token|key|secret|password|auth|cookie/iu.test(key)) url.searchParams.set(key, "[redacted]");
}
return url.toString();
} catch {
return truncate(String(value || ""), 300);
}
}
function normalizeBaseUrl(value) {
const raw = value || "http://127.0.0.1:3000";
const url = new URL(raw);
return url.origin;
}
function parseViewport(value) {
const match = String(value).match(/^(\d{3,5})x(\d{3,5})$/u);
return match ? { width: Number(match[1]), height: Number(match[2]) } : { width: 1440, height: 900 };
}
function positiveInteger(value, fallback) {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallback;
}
function sha256Text(value) {
return "sha256:" + createHash("sha256").update(String(value)).digest("hex");
}
function safeId(value) {
return String(value || "").replace(/[^A-Za-z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "").slice(0, 120) || "item";
}
function cssEscape(value) {
return String(value).replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"");
}
function truncate(value, limit) {
const text = String(value || "");
return text.length > limit ? text.slice(0, limit) + "..." : text;
}
function sanitize(value) {
if (value === null || value === undefined) return value;
if (typeof value === "string") return value === password ? "[redacted]" : value.replaceAll(password, "[redacted]");
if (typeof value === "number" || typeof value === "boolean") return value;
if (Array.isArray(value)) return value.map(sanitize);
if (typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => /password|cookie|authorization|token|secret/iu.test(key) ? [key, "[redacted]"] : [key, sanitize(item)]));
return String(value);
}
function errorSummary(error) {
return { name: error && error.name ? error.name : "Error", message: error && error.message ? truncate(error.message, 1000) : truncate(String(error), 1000), stackTail: error && error.stack ? truncate(error.stack, 2000) : null };
}
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
}
`;
}
export function nodeWebObserveAnalyzerSource(): string {
return String.raw`#!/usr/bin/env node
import { createHash } from "node:crypto";
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
import path from "node:path";
const stateDir = path.resolve(process.env.UNIDESK_WEB_OBSERVE_STATE_DIR || process.argv[2] || ".state/web-observe/manual");
const analysisDir = path.join(stateDir, "analysis");
const reportJsonPath = path.join(analysisDir, "report.json");
const reportMdPath = path.join(analysisDir, "report.md");
const samples = await readJsonl(path.join(stateDir, "samples.jsonl"));
const control = await readJsonl(path.join(stateDir, "control.jsonl"));
const network = await readJsonl(path.join(stateDir, "network.jsonl"));
const errors = await readJsonl(path.join(stateDir, "errors.jsonl"));
const artifacts = await readJsonl(path.join(stateDir, "artifacts.jsonl"));
const manifest = await readJson(path.join(stateDir, "manifest.json"));
const heartbeat = await readJson(path.join(stateDir, "heartbeat.json"));
await mkdir(analysisDir, { recursive: true, mode: 0o700 });
const findings = buildFindings(samples, control, network, errors);
const transitions = buildTransitions(samples);
const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl }));
const report = {
ok: findings.filter((item) => item.severity === "red").length === 0,
command: "web-probe-observe analyze",
generatedAt: new Date().toISOString(),
stateDir,
manifest: compactManifest(manifest),
heartbeat: compactHeartbeat(heartbeat),
counts: { samples: samples.length, control: control.length, network: network.length, errors: errors.length, artifacts: artifacts.length },
commandTimeline,
transitions,
findings,
artifactSummary: await artifactSummary(artifacts),
safety: { offlineOnly: true, browserDriven: false, apiFetch: false, valuesRedacted: true },
};
await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 });
await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 });
const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]);
console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, findings: findings.slice(0, 20), valuesRedacted: true }, null, 2));
async function readJson(file) {
try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; }
}
async function readJsonl(file) {
try {
const text = await readFile(file, "utf8");
return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => {
try { return JSON.parse(line); } catch { return { parseError: true, rawHash: sha256(line) }; }
});
} catch { return []; }
}
function buildFindings(samples, control, network, errors) {
const findings = [];
const commandTimes = control.filter((item) => item.phase === "completed" || item.phase === "started").map((item) => Date.parse(item.ts)).filter(Number.isFinite);
const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean));
const activeSessions = new Set(samples.map((item) => item.activeSessionId).filter(Boolean));
if (routeSessions.size > 1) findings.push({ id: "session-route-changed", severity: "amber", summary: "route session changed during observation", routeSessionCount: routeSessions.size, samples: sampleRefs(samples, (item) => item.routeSessionId) });
if (activeSessions.size > 1) findings.push({ id: "active-session-changed", severity: "amber", summary: "active session changed during observation", activeSessionCount: activeSessions.size, samples: sampleRefs(samples, (item) => item.activeSessionId) });
const mismatches = samples.filter((item) => item.routeSessionId && item.activeSessionId && item.routeSessionId !== item.activeSessionId);
if (mismatches.length > 0) findings.push({ id: "route-active-session-mismatch", severity: "red", summary: "routeSessionId and activeSessionId diverged", count: mismatches.length, samples: mismatches.slice(0, 10).map(ref) });
const uncommandedChanges = [];
for (let i = 1; i < samples.length; i += 1) {
const prev = digestSample(samples[i - 1]);
const next = digestSample(samples[i]);
if (prev !== next && !nearCommand(samples[i], commandTimes, 10000)) uncommandedChanges.push(ref(samples[i]));
}
if (uncommandedChanges.length > 0) findings.push({ id: "uncommanded-visible-state-change", severity: "amber", summary: "visible message/trace digest changed without a nearby command", count: uncommandedChanges.length, samples: uncommandedChanges.slice(0, 20) });
const finalFlicker = detectFinalFlicker(samples);
if (finalFlicker.length > 0) findings.push({ id: "final-response-flicker", severity: "red", summary: "message text digest disappeared or switched to diagnostic-like text after non-empty final text", count: finalFlicker.length, samples: finalFlicker.slice(0, 20) });
const scrollJumps = [];
for (let i = 1; i < samples.length; i += 1) {
const prevY = Number(samples[i - 1]?.scroll?.y ?? 0);
const nextY = Number(samples[i]?.scroll?.y ?? 0);
if (prevY > 250 && nextY < 40 && !nearCommand(samples[i], commandTimes, 8000)) scrollJumps.push({ from: ref(samples[i - 1]), to: ref(samples[i]) });
}
if (scrollJumps.length > 0) findings.push({ id: "scroll-jump-top", severity: "amber", summary: "scroll position jumped near top without nearby command", count: scrollJumps.length, samples: scrollJumps.slice(0, 10) });
const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => /completed|failed|canceled|terminal|done/iu.test((row.status || "") + " " + (row.textPreview || ""))));
const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0);
if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) });
const naturalApi = network.filter((item) => item.observerInitiated === false && item.type === "response" && /\/v1\/|\/auth\//u.test(String(item.url || "")));
findings.push({ id: "natural-api-dom-lag-baseline", severity: "info", summary: "natural API responses and DOM samples are available for manual lag correlation", naturalApiResponses: naturalApi.length, sampleCount: samples.length });
if (errors.length > 0) findings.push({ id: "browser-console-or-page-errors", severity: "amber", summary: "pageerror/runner errors were captured", count: errors.length, first: errors.slice(0, 5) });
if (samples.length === 0) findings.push({ id: "no-samples", severity: "red", summary: "observer produced no samples" });
return findings;
}
function buildTransitions(samples) {
const rows = [];
let last = null;
for (const sample of samples) {
const digest = digestSample(sample);
if (digest !== last) {
rows.push({ seq: sample.seq, ts: sample.ts, url: sample.url, routeSessionId: sample.routeSessionId || null, activeSessionId: sample.activeSessionId || null, messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0, traceRowCount: Array.isArray(sample.traceRows) ? sample.traceRows.length : 0, digest });
last = digest;
}
}
return rows.slice(0, 200);
}
function detectFinalFlicker(samples) {
const flickers = [];
let lastNonEmpty = null;
for (const sample of samples) {
const messageText = Array.isArray(sample.messages) ? sample.messages.map((item) => item.textPreview || "").join("\n") : "";
const nonEmpty = messageText.trim().length > 0;
if (nonEmpty && !/temporarily|timeout|无法连接|暂时|error|failed|超时/iu.test(messageText)) lastNonEmpty = { sample, messageText };
if (lastNonEmpty && nonEmpty && /temporarily|timeout|无法连接|暂时|error|failed|超时/iu.test(messageText)) flickers.push({ from: ref(lastNonEmpty.sample), to: ref(sample) });
if (lastNonEmpty && !nonEmpty) flickers.push({ from: ref(lastNonEmpty.sample), to: ref(sample), reason: "text-disappeared" });
}
return flickers;
}
function digestSample(sample) {
const messages = Array.isArray(sample.messages) ? sample.messages.map((item) => item.textHash || item.textPreview || "").join("|") : "";
const trace = Array.isArray(sample.traceRows) ? sample.traceRows.map((item) => (item.status || "") + ":" + (item.textHash || item.textPreview || "")).join("|") : "";
return sha256((sample.routeSessionId || "") + "|" + (sample.activeSessionId || "") + "|" + messages + "|" + trace);
}
function nearCommand(sample, commandTimes, windowMs) {
const ts = Date.parse(sample.ts);
return Number.isFinite(ts) && commandTimes.some((item) => Math.abs(ts - item) <= windowMs);
}
function sampleRefs(samples, pick) {
const seen = new Set();
const refs = [];
for (const sample of samples) {
const value = pick(sample);
if (!value || seen.has(value)) continue;
seen.add(value);
refs.push({ ...ref(sample), value });
}
return refs.slice(0, 20);
}
function ref(sample) {
if (!sample) return null;
return { seq: sample.seq ?? null, ts: sample.ts ?? null, url: sample.url ?? null, routeSessionId: sample.routeSessionId ?? null, activeSessionId: sample.activeSessionId ?? null };
}
async function artifactSummary(artifacts) {
const items = artifacts.slice(-30).map((item) => ({ kind: item.kind, reason: item.reason, sampleSeq: item.sampleSeq, path: item.path, sha256: item.sha256, byteCount: item.byteCount }));
return { count: artifacts.length, latest: items };
}
function compactManifest(value) {
if (!value) return null;
return { jobId: value.jobId, stateDir: value.stateDir, baseUrl: value.baseUrl, targetPath: value.targetPath, startedAt: value.startedAt, status: value.status, sampling: value.sampling, safety: value.safety };
}
function compactHeartbeat(value) {
if (!value) return null;
return { jobId: value.jobId, pid: value.pid, status: value.status, sampleSeq: value.sampleSeq, commandSeq: value.commandSeq, currentUrl: value.currentUrl, updatedAt: value.updatedAt, uptimeMs: value.uptimeMs };
}
function renderMarkdown(report) {
const findingLines = report.findings.length === 0 ? "- 无红灯项。" : report.findings.map((item) => "- " + item.severity + ": " + item.id + " - " + item.summary).join("\n");
const commandLines = report.commandTimeline.length === 0 ? "- 无控制命令。" : report.commandTimeline.map((item) => "- " + item.ts + " " + item.phase + " " + item.type + " " + item.commandId + " " + (item.afterUrl || "")).join("\n");
const transitionLines = report.transitions.length === 0 ? "- 无状态变化。" : report.transitions.slice(0, 80).map((item) => "- #" + item.seq + " " + item.ts + " messages=" + item.messageCount + " traceRows=" + item.traceRowCount + " route=" + (item.routeSessionId || "-") + " active=" + (item.activeSessionId || "-")).join("\n");
return "# web-probe observe analysis\n\n"
+ "- stateDir: " + report.stateDir + "\n"
+ "- generatedAt: " + report.generatedAt + "\n"
+ "- samples: " + report.counts.samples + "\n"
+ "- control: " + report.counts.control + "\n"
+ "- network: " + report.counts.network + "\n"
+ "- errors: " + report.counts.errors + "\n\n"
+ "## Findings\n\n" + findingLines + "\n\n"
+ "## Command timeline\n\n" + commandLines + "\n\n"
+ "## State transitions\n\n" + transitionLines + "\n";
}
async function fileMeta(file) {
const [buffer, stats] = await Promise.all([readFile(file), stat(file)]);
return { byteCount: stats.size, sha256: "sha256:" + createHash("sha256").update(buffer).digest("hex") };
}
function sha256(value) {
return "sha256:" + createHash("sha256").update(String(value)).digest("hex");
}
`;
}