diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index 5e9a83ea..67bdc1cb 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -74,8 +74,8 @@ export function hwlabNodeWebProbeHelp(): Record { examples: [ "bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --wait-messages-ms 1000", "bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --url https://hwlab.pikapython.com --fresh-session --message 'ping'", - "bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 --script-file .state/probes/workbench.mjs", - "bun scripts/cli.ts hwlab nodes web-probe observe start --node D601 --lane v03 --target-path /workbench --sample-interval-ms 5000", + "bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 --browser-proxy-mode direct --script-file .state/probes/workbench.mjs", + "bun scripts/cli.ts hwlab nodes web-probe observe start --node D601 --lane v03 --target-path /workbench --sample-interval-ms 5000 --browser-proxy-mode direct", "bun scripts/cli.ts hwlab nodes web-probe observe command webobs-xxxx --type newSession", "bun scripts/cli.ts hwlab nodes web-probe observe command webobs-xxxx --type selectProvider --provider codex-api", "bun scripts/cli.ts hwlab nodes web-probe observe command webobs-xxxx --type sendPrompt --text 'ping'", @@ -97,6 +97,7 @@ export function hwlabNodeWebProbeHelp(): Record { "observe analyze is offline-only: it reads artifact JSONL and writes analysis/report.md plus analysis/report.json without accessing Workbench APIs or driving the browser.", "observe analyze scans every sampled DOM point, extracts Workbench timing text such as 总耗时/total and 最近 N 秒/分前, and writes a sample point vs turn timing report: each Markdown table row starts with the timestamp, followed by each turn's 总耗时(s) and 最近更新(s). Timing series are reported for post-processing/manual analysis instead of auto-judged from status tail output.", "observe analyze also reports visible “加载中” count, owner attribution, concurrent loading owners, and continuous visible segments; fixes must reduce real loading latency, not reveal incomplete content early to make this metric disappear.", + "script/observe support --browser-proxy-mode auto|direct; use direct for A/B evidence when frontend RUM is slow but OTel server spans are absent or fast, instead of falling back to raw Playwright.", "Use recordStep(name, data) or fetchApiMatrix(paths) to keep structured partial evidence when a later step fails.", "Use reloadStable(), gotoCurrentStable(), or safeReload() for bounded retries around page reload/current-URL navigation jitter such as ERR_NETWORK_CHANGED.", "Playwright page.evaluate accepts one serializable argument; use page.evaluate(({ a, b }) => ..., { a, b }) or safeEvaluate(fn, { a, b }).", diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index aa1e7cd4..212c995d 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -20,6 +20,7 @@ import type { RenderedCliResult } from "./output"; type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete"; type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup"; type NodeRuntimeRenderLocation = "node-host" | "local"; +type WebProbeBrowserProxyMode = "auto" | "direct"; interface NodeWebProbeRunOptions { action: "run"; @@ -48,6 +49,7 @@ interface NodeWebProbeScriptOptions { url: string; timeoutMs: number; viewport: string; + browserProxyMode: WebProbeBrowserProxyMode; commandTimeoutSeconds: number; scriptText: string; scriptSource: { @@ -70,6 +72,7 @@ interface NodeWebProbeObserveOptions { url: string; targetPath: string; viewport: string; + browserProxyMode: WebProbeBrowserProxyMode; sampleIntervalMs: number; screenshotIntervalMs: number; observerRefreshIntervalMs: number; @@ -6774,6 +6777,7 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions { "--url", "--timeout-ms", "--viewport", + "--browser-proxy-mode", "--script-file", "--command-timeout-seconds", ]), new Set([])); @@ -6792,6 +6796,7 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions { url: optionValue(args, "--url") ?? spec.publicWebUrl, timeoutMs, viewport: optionValue(args, "--viewport") ?? "1440x900", + browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode")), commandTimeoutSeconds, scriptText, scriptSource: { @@ -6899,6 +6904,7 @@ function parseNodeWebProbeObserveOptions( "--url", "--target-path", "--viewport", + "--browser-proxy-mode", "--sample-interval-ms", "--screenshot-interval-ms", "--observer-refresh-interval-ms", @@ -6940,6 +6946,7 @@ function parseNodeWebProbeObserveOptions( url: optionValue(args, "--url") ?? spec.publicWebUrl, targetPath: optionValue(args, "--target-path") ?? "/workbench", viewport: optionValue(args, "--viewport") ?? "1440x900", + browserProxyMode: parseWebProbeBrowserProxyMode(optionValue(args, "--browser-proxy-mode")), sampleIntervalMs: positiveIntegerOption(args, "--sample-interval-ms", 5000, 600000), screenshotIntervalMs: positiveIntegerOption(args, "--screenshot-interval-ms", 300000, 86_400_000), observerRefreshIntervalMs: positiveIntegerOption(args, "--observer-refresh-interval-ms", 180000, 86_400_000), @@ -6979,6 +6986,12 @@ function parseNodeWebProbeObserveCommandType(value: string): NodeWebProbeObserve throw new Error(`web-probe observe command --type must be login, preflight, goto, newSession, sendPrompt, selectProvider, clickSession, screenshot, mark, or stop; got ${value}`); } +function parseWebProbeBrowserProxyMode(value: string | undefined): WebProbeBrowserProxyMode { + if (value === undefined || value === "auto") return "auto"; + if (value === "direct") return "direct"; + throw new Error(`web-probe --browser-proxy-mode must be auto or direct, got ${value}`); +} + function nodeWebProbeAutoCommandTimeoutSeconds(input: { timeoutMs: number; waitAfterSubmitMs: number; @@ -7421,6 +7434,7 @@ function runNodeWebProbeObserveStart( `UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS=${shellQuote(String(options.observerRefreshIntervalMs))}`, `UNIDESK_WEB_OBSERVE_MAX_SAMPLES=${shellQuote(String(options.maxSamples))}`, `UNIDESK_WEB_OBSERVE_VIEWPORT=${shellQuote(options.viewport)}`, + `UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`, ].join(" "); const script = [ "set -eu", @@ -9505,23 +9519,24 @@ function runNodeWebProbeScript( const webProbeProxy = nodeWebProbeHostProxyEnv(spec); const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? "", webProbeProxy); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); + const commandTimedOut = result.timedOut || result.exitCode === 124; const stdoutReport = parseJsonObject(result.stdout); const runPaths = webProbeScriptRunPathsFromStderr(result.stderr); const recoveredReport = stdoutReport === null ? readNodeWebProbeScriptReport(options, spec, runPaths.reportPath) : null; - const recoveredArtifacts = stdoutReport === null || result.timedOut ? readNodeWebProbeScriptArtifacts(options, spec, runPaths.runDir) : null; + const recoveredArtifacts = stdoutReport === null || commandTimedOut ? readNodeWebProbeScriptArtifacts(options, spec, runPaths.runDir) : null; const parsedReport = stdoutReport ?? recoveredReport?.report ?? null; const report = compactWebProbeScriptResult(parsedReport); const passed = result.exitCode === 0 && report?.ok === true; const summary = nullableRecord(report?.summary); const stdoutBytes = Buffer.byteLength(result.stdout, "utf8"); const outputFailureKind = parsedReport === null - ? result.timedOut + ? commandTimedOut ? "web-probe-command-timeout" : stdoutBytes > 64 * 1024 ? "web-probe-output-too-large" : "web-probe-report-parse-failed" : null; - const degradedReason = result.timedOut + const degradedReason = commandTimedOut ? "web-probe-command-timeout" : typeof summary?.degradedReason === "string" ? summary.degradedReason @@ -9529,10 +9544,10 @@ function runNodeWebProbeScript( ? report.failureKind : outputFailureKind ? outputFailureKind - : result.timedOut + : commandTimedOut ? "web-probe-command-timeout" : null; - const failureKind = result.timedOut + const failureKind = commandTimedOut ? "web-probe-command-timeout" : typeof summary?.failureKind === "string" ? summary.failureKind @@ -9541,7 +9556,7 @@ function runNodeWebProbeScript( : outputFailureKind; const effectiveSummary = summary !== null ? { ...summary, - transportTimedOut: result.timedOut, + transportTimedOut: commandTimedOut, recoveredFrom: stdoutReport !== null ? "stdout" : recoveredReport?.source ?? null, } : (outputFailureKind === null ? null : { ok: false, @@ -9564,6 +9579,7 @@ function runNodeWebProbeScript( screenshots: recoveredArtifacts?.screenshots ?? [], artifacts: recoveredArtifacts?.artifacts ?? null, stdoutBytes, + exitCode: result.exitCode, stderrTail: result.stderr.trim().slice(-2000), valuesRedacted: true, }); @@ -9631,6 +9647,7 @@ function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secre "UNIDESK_WEB_PROBE_USER_SCRIPT=\"$user_script\"", `UNIDESK_WEB_PROBE_TIMEOUT_MS=${shellQuote(String(options.timeoutMs))}`, `UNIDESK_WEB_PROBE_VIEWPORT=${shellQuote(options.viewport)}`, + `UNIDESK_WEB_PROBE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`, "node \"$runner\"", ].join(" "), ].join("\n"); diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index a4be7252..b73cbf48 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -22,6 +22,7 @@ const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCR const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0); const observerRefreshIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_OBSERVER_REFRESH_INTERVAL_MS, 180000); const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900"); +const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE || "auto"); const playwrightProxy = proxyConfigFromEnv(baseUrl); const chromiumLaunchOptions = chromiumLaunchOptionsForProxy(playwrightProxy); const pageId = "control-" + randomBytes(4).toString("hex"); @@ -512,9 +513,9 @@ async function authenticate(browserContext, authPage) { async function pageAuthLogin(authPage, loginUrl) { if (!authPage) throw new Error("auth page is not ready"); - await authPage.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: 12000 }); + await authPage.goto(new URL("/assets/favicon.svg", baseUrl).toString(), { waitUntil: "domcontentloaded", timeout: 12000 }); return authPage.evaluate(async (input) => { - const response = await fetch("/auth/login", { + const response = await fetch(input.loginUrl, { method: "POST", headers: { accept: "application/json", "content-type": "application/json" }, body: JSON.stringify({ username: input.username, password: input.password }), @@ -526,7 +527,7 @@ async function pageAuthLogin(authPage, loginUrl) { status: response.status, statusText: response.statusText || "", }; - }, { username, password }); + }, { username, password, loginUrl }); } function publicAuth(value) { @@ -560,7 +561,7 @@ function isRetryableAuthStatus(status) { function isRetryableAuthError(error) { const message = error && error.message ? String(error.message) : String(error || ""); - return /AbortError|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|ECONNABORTED|socket hang up|ERR_NETWORK_CHANGED|fetch failed|network|timeout|aborted/iu.test(message); + return /AbortError|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|ECONNABORTED|socket hang up|ERR_NETWORK_CHANGED|fetch failed|failed to fetch|network|timeout|aborted/iu.test(message); } function authFailureMessage(failure) { @@ -573,6 +574,7 @@ function authFailureMessage(failure) { } function proxyConfigFromEnv(targetBaseUrl) { + if (browserProxyMode === "direct") return null; let target; try { target = new URL(targetBaseUrl); @@ -628,6 +630,7 @@ function publicNetwork(proxy) { }, browser: { proxyMode: proxy === null ? "direct-no-proxy-server" : "explicit-playwright-proxy", + requestedProxyMode: browserProxyMode, proxyEnvCleared: true, valuesRedacted: true, }, @@ -635,6 +638,11 @@ function publicNetwork(proxy) { }; } +function parseBrowserProxyMode(raw) { + if (raw === "auto" || raw === "direct") return raw; + return "auto"; +} + function publicProxyServer(raw) { try { const parsed = new URL(String(raw || "")); @@ -2383,10 +2391,10 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN const sessionRailTitleSummary = sampleMetrics?.sessionRailTitles?.summary || {}; if (Number(sessionRailTitleSummary.majorityFallbackSampleCount ?? 0) > 0) findings.push({ id: "session-rail-title-fallback-majority", severity: "red", summary: "more than half of visible session list rows used fallback Session ses_... titles", count: sessionRailTitleSummary.majorityFallbackSampleCount, maxFallbackRatio: sessionRailTitleSummary.maxFallbackRatio, maxFallbackTitleCount: sessionRailTitleSummary.maxFallbackTitleCount, samples: sampleMetrics.sessionRailTitles.samples.slice(0, 20), examples: sampleMetrics.sessionRailTitles.examples.slice(0, 20) }); if ((runtimeAlerts?.summary?.httpErrorCount ?? 0) > 0) findings.push({ id: "runtime-http-errors", severity: "amber", summary: "natural page requests returned HTTP error status during observation", count: runtimeAlerts.summary.httpErrorCount, groups: runtimeAlerts.networkHttpErrorsByPath.slice(0, 12) }); - if ((runtimeAlerts?.summary?.requestFailedCount ?? 0) > 0) findings.push({ id: "runtime-requestfailed", severity: "amber", summary: "browser requestfailed events were captured during observation", count: runtimeAlerts.summary.requestFailedCount, groups: runtimeAlerts.networkRequestFailedByPath.slice(0, 12) }); + if ((runtimeAlerts?.summary?.significantRequestFailedCount ?? runtimeAlerts?.summary?.requestFailedCount ?? 0) > 0) findings.push({ id: "runtime-requestfailed", severity: "amber", summary: "browser requestfailed events were captured during observation", count: runtimeAlerts.summary.significantRequestFailedCount ?? runtimeAlerts.summary.requestFailedCount, groups: (runtimeAlerts.networkSignificantRequestFailedByPath ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 12) }); if ((runtimeAlerts?.summary?.domDiagnosticSampleCount ?? 0) > 0) findings.push({ id: "runtime-dom-diagnostics", severity: "amber", summary: "diagnostic/error/warning-like text was visible in sampled DOM", count: runtimeAlerts.summary.domDiagnosticSampleCount, groupCount: runtimeAlerts.summary.domDiagnosticGroupCount ?? 0, groups: runtimeAlerts.domDiagnosticsByText.slice(0, 12), samples: runtimeAlerts.domDiagnostics.slice(0, 12) }); if ((runtimeAlerts?.summary?.executionErrorCount ?? 0) > 0) findings.push({ id: "runtime-execution-errors", severity: "red", summary: "Workbench rendered execution failure/error rows during observation", count: runtimeAlerts.summary.executionErrorCount, groups: runtimeAlerts.runtimeExecutionErrorsByCode.slice(0, 12) }); - if ((runtimeAlerts?.summary?.consoleAlertCount ?? 0) > 0) findings.push({ id: "runtime-console-alerts", severity: "amber", summary: "browser console warning/error entries were captured during observation", count: runtimeAlerts.summary.consoleAlertCount, groups: runtimeAlerts.consoleAlertsByPath.slice(0, 12) }); + if ((runtimeAlerts?.summary?.significantConsoleAlertCount ?? runtimeAlerts?.summary?.consoleAlertCount ?? 0) > 0) findings.push({ id: "runtime-console-alerts", severity: "amber", summary: "browser console warning/error entries were captured during observation", count: runtimeAlerts.summary.significantConsoleAlertCount ?? runtimeAlerts.summary.consoleAlertCount, groups: (runtimeAlerts.significantConsoleAlertsByPath ?? runtimeAlerts.consoleAlertsByPath).slice(0, 12) }); const crossPageDiffs = detectCrossPageProjectionDiffs(samples); const crossPageProjectionDiffs = crossPageDiffs.filter((item) => item.diffKind !== "trace-visibility"); const crossPageTraceVisibilityDiffs = crossPageDiffs.filter((item) => item.diffKind === "trace-visibility"); @@ -2946,6 +2954,7 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) { const requestFailed = naturalNetwork .filter((item) => item?.type === "requestfailed") .map((item) => networkAlertEvent(item, promptTimes)); + const significantRequestFailed = requestFailed.filter((item) => !isBenignLongLivedStreamClosureAlert(item)); const domDiagnostics = []; const executionErrors = []; const baselineExecutionErrors = []; @@ -3049,6 +3058,7 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) { const consoleAlerts = consoleEvents .filter((item) => /error|warning|warn|assert/iu.test(String(item?.type || "")) || isDiagnosticText(item?.text)) .map((item) => consoleAlertEvent(item, promptTimes)); + const significantConsoleAlerts = consoleAlerts.filter((item) => !isBenignLongLivedStreamClosureAlert(item)); const pageErrors = errors.map((item) => ({ ts: item.ts ?? null, promptIndex: promptIndexForTs(promptTimes, item.ts), @@ -3061,20 +3071,26 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) { summary: { httpErrorCount: httpErrors.length, requestFailedCount: requestFailed.length, + significantRequestFailedCount: significantRequestFailed.length, + benignLongLivedStreamClosureCount: requestFailed.length - significantRequestFailed.length, domDiagnosticSampleCount: domDiagnostics.length, domDiagnosticGroupCount: groupDomDiagnostics(domDiagnostics).length, executionErrorCount: executionErrors.length, baselineExecutionErrorCount: baselineExecutionErrors.length, consoleAlertCount: consoleAlerts.length, + significantConsoleAlertCount: significantConsoleAlerts.length, pageErrorCount: pageErrors.length, networkErrorGroupCount: groupNetworkAlerts(httpErrors).length, requestFailedGroupCount: groupNetworkAlerts(requestFailed).length, + significantRequestFailedGroupCount: groupNetworkAlerts(significantRequestFailed).length, executionErrorGroupCount: groupExecutionErrors(executionErrors).length, baselineExecutionErrorGroupCount: groupExecutionErrors(baselineExecutionErrors).length, - consoleAlertGroupCount: groupConsoleAlerts(consoleAlerts).length + consoleAlertGroupCount: groupConsoleAlerts(consoleAlerts).length, + significantConsoleAlertGroupCount: groupConsoleAlerts(significantConsoleAlerts).length }, networkHttpErrorsByPath: groupNetworkAlerts(httpErrors), networkRequestFailedByPath: groupNetworkAlerts(requestFailed), + networkSignificantRequestFailedByPath: groupNetworkAlerts(significantRequestFailed), domDiagnostics: domDiagnostics.slice(-80), domDiagnosticsByText: groupDomDiagnostics(domDiagnostics), domDiagnosticsByFingerprint: groupDomDiagnostics(domDiagnostics).slice(0, 80), @@ -3084,6 +3100,8 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) { baselineRuntimeExecutionErrorsByCode: groupExecutionErrors(baselineExecutionErrors), consoleAlerts: consoleAlerts.slice(0, 80), consoleAlertsByPath: groupConsoleAlerts(consoleAlerts), + significantConsoleAlerts: significantConsoleAlerts.slice(0, 80), + significantConsoleAlertsByPath: groupConsoleAlerts(significantConsoleAlerts), pageErrors: pageErrors.slice(0, 40) }; } @@ -3313,6 +3331,13 @@ function groupConsoleAlerts(events) { return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(a.urlPath).localeCompare(String(b.urlPath))); } +function isBenignLongLivedStreamClosureAlert(event) { + if (event?.urlPath !== "/v1/workbench/events") return false; + if (event.status !== null && event.status !== undefined && Number(event.status) > 0) return false; + const text = String(event.failureKind || event.errorPreview || event.preview || ""); + return /ERR_NETWORK_CHANGED|ERR_ABORTED|net::ERR_NETWORK_CHANGED|net::ERR_ABORTED|aborted|network changed/iu.test(text); +} + function networkAlertEvent(item, promptTimes) { const failureText = item.failureKind ?? item.failure ?? item.errorText ?? null; return { diff --git a/scripts/src/hwlab-node-web-probe-runner-source.ts b/scripts/src/hwlab-node-web-probe-runner-source.ts index 3d586935..693b2c86 100644 --- a/scripts/src/hwlab-node-web-probe-runner-source.ts +++ b/scripts/src/hwlab-node-web-probe-runner-source.ts @@ -17,6 +17,7 @@ const runDir = path.resolve(process.env.UNIDESK_WEB_PROBE_RUN_DIR || ".state/web const userScript = path.resolve(process.env.UNIDESK_WEB_PROBE_USER_SCRIPT || path.join(runDir, "user-script.mjs")); const timeoutMs = positiveInteger(process.env.UNIDESK_WEB_PROBE_TIMEOUT_MS, 30000); const viewport = parseViewport(process.env.UNIDESK_WEB_PROBE_VIEWPORT || "1440x900"); +const browserProxyMode = parseBrowserProxyMode(process.env.UNIDESK_WEB_PROBE_BROWSER_PROXY_MODE || "auto"); const playwrightProxy = proxyConfigFromEnv(baseUrl); const chromiumLaunchOptions = chromiumLaunchOptionsForProxy(playwrightProxy); const artifactRecords = []; @@ -370,6 +371,7 @@ function scriptHelpers() { } function proxyConfigFromEnv(targetBaseUrl) { + if (browserProxyMode === "direct") return null; let target; try { target = new URL(targetBaseUrl); @@ -425,6 +427,7 @@ function publicNetwork(proxy) { }, browser: { proxyMode: proxy === null ? "direct-no-proxy-server" : "explicit-playwright-proxy", + requestedProxyMode: browserProxyMode, proxyEnvCleared: true, valuesRedacted: true, }, @@ -432,6 +435,11 @@ function publicNetwork(proxy) { }; } +function parseBrowserProxyMode(raw) { + if (raw === "auto" || raw === "direct") return raw; + return "auto"; +} + function publicProxyServer(raw) { try { const parsed = new URL(String(raw || ""));