diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index 5b72dc6f..eadd88f0 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -68,12 +68,13 @@ export function hwlabNodeWebProbeHelp(): Record { ], actions: { run: "Run the repo-owned scripts/web-live-dom-probe.mjs helper.", - script: "Run caller-provided Playwright JS after CLI-managed /auth/login; the script receives authenticated browser/context/page plus fetchJson/safeFetchJson/fetchApiMatrix/recordStep/collectText/safeEvaluate/waitWorkbenchReady/screenshotOnError/summarizeWorkspace/summarizeConversation helpers and must not handle secrets itself.", + script: "Run caller-provided Playwright JS after CLI-managed /auth/login; the script receives authenticated browser/context/page plus gotoStable/reloadStable/gotoCurrentStable/safeReload/fetchJson/safeFetchJson/fetchApiMatrix/recordStep/collectText/safeEvaluate/waitWorkbenchReady/screenshotOnError/summarizeWorkspace/summarizeConversation helpers and must not handle secrets itself.", }, notes: [ "Prefer --script-file for reusable probes; stdin heredocs remain supported for one-off probes.", "Issue-ready summary is available under probe.summary; full script report is persisted under probe.reportPath with a SHA-256 fingerprint.", "Use recordStep(name, data) or fetchApiMatrix(paths) to keep structured partial evidence when a later step fails.", + "Use reloadStable(), gotoCurrentStable(), or safeReload() for bounded retries around page reload/current-URL navigation jitter such as ERR_NETWORK_CHANGED.", "Playwright page.evaluate accepts one serializable argument; use page.evaluate(({ a, b }) => ..., { a, b }) or safeEvaluate(fn, { a, b }).", "Failures include failureKind, errorMessage, scriptSha256, runDir, lastUrl, and lastScreenshot when a screenshot can be captured.", ], diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index b48ef1ff..f0568f30 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -5131,17 +5131,25 @@ function runNodeWebProbeScript( ): Record { const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? ""); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); - const parsedReport = parseJsonObject(result.stdout); + const stdoutReport = parseJsonObject(result.stdout); + const runPaths = webProbeScriptRunPathsFromStderr(result.stderr); + const recoveredReport = stdoutReport === null ? readNodeWebProbeScriptReport(options, spec, runPaths.reportPath) : null; + const recoveredArtifacts = stdoutReport === null || result.timedOut ? readNodeWebProbeScriptArtifacts(options, spec, runPaths.runDir) : null; + const parsedReport = stdoutReport ?? recoveredReport?.report ?? null; const report = compactWebProbeScriptResult(parsedReport); const passed = result.exitCode === 0 && report?.ok === true; const summary = nullableRecord(report?.summary); const stdoutBytes = Buffer.byteLength(result.stdout, "utf8"); const outputFailureKind = parsedReport === null - ? stdoutBytes > 64 * 1024 + ? result.timedOut + ? "web-probe-command-timeout" + : stdoutBytes > 64 * 1024 ? "web-probe-output-too-large" : "web-probe-report-parse-failed" : null; - const degradedReason = typeof summary?.degradedReason === "string" + const degradedReason = result.timedOut + ? "web-probe-command-timeout" + : typeof summary?.degradedReason === "string" ? summary.degradedReason : typeof report?.failureKind === "string" ? report.failureKind @@ -5150,19 +5158,37 @@ function runNodeWebProbeScript( : result.timedOut ? "web-probe-command-timeout" : null; - const failureKind = typeof summary?.failureKind === "string" + const failureKind = result.timedOut + ? "web-probe-command-timeout" + : typeof summary?.failureKind === "string" ? summary.failureKind : typeof report?.failureKind === "string" ? report.failureKind : outputFailureKind; - const effectiveSummary = summary ?? (outputFailureKind === null ? null : { + const effectiveSummary = summary !== null ? { + ...summary, + transportTimedOut: result.timedOut, + recoveredFrom: stdoutReport !== null ? "stdout" : recoveredReport?.source ?? null, + } : (outputFailureKind === null ? null : { ok: false, status: "blocked", degradedReason: outputFailureKind, failureKind: outputFailureKind, failedCondition: outputFailureKind === "web-probe-output-too-large" ? "remote web-probe stdout exceeded the bounded summary parser input" + : outputFailureKind === "web-probe-command-timeout" + ? "remote web-probe command timed out before stdout returned a parseable report" : "remote web-probe stdout did not contain a parseable JSON report", + runDir: runPaths.runDir, + reportPath: runPaths.reportPath, + reportLoad: recoveredReport === null ? null : { + source: recoveredReport.source, + path: recoveredReport.path, + degradedReason: recoveredReport.degradedReason, + result: recoveredReport.result === null ? null : compactCommandResultRedacted(recoveredReport.result, [material.password ?? ""]), + }, + screenshots: recoveredArtifacts?.screenshots ?? [], + artifacts: recoveredArtifacts?.artifacts ?? null, stdoutBytes, stderrTail: result.stderr.trim().slice(-2000), valuesRedacted: true, @@ -5186,6 +5212,18 @@ function runNodeWebProbeScript( failureKind, summary: effectiveSummary, probe: report, + reportLoad: stdoutReport !== null ? { source: "stdout", path: report?.reportPath ?? null, degradedReason: null } : recoveredReport === null ? null : { + source: recoveredReport.source, + path: recoveredReport.path, + degradedReason: recoveredReport.degradedReason, + result: recoveredReport.result === null ? null : compactCommandResultRedacted(recoveredReport.result, [material.password ?? ""]), + }, + recoveredArtifacts: recoveredArtifacts === null ? null : { + source: recoveredArtifacts.source, + degradedReason: recoveredArtifacts.degradedReason, + result: recoveredArtifacts.result === null ? null : compactCommandResultRedacted(recoveredArtifacts.result, [material.password ?? ""]), + artifacts: recoveredArtifacts.artifacts, + }, result: compactResult, valuesRedacted: true, }; @@ -5199,7 +5237,10 @@ function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secre "run_root=.state/web-probe-script", "mkdir -p \"$run_root\"", "run_dir=$(mktemp -d \"$run_root/run.XXXXXX\")", + "report_file=\"$run_dir/web-probe-script-report.json\"", "chmod 700 \"$run_dir\"", + "printf 'UNIDESK_WEB_PROBE_RUN_DIR=%s\\n' \"$run_dir\" >&2", + "printf 'UNIDESK_WEB_PROBE_REPORT_PATH=%s\\n' \"$report_file\" >&2", "user_script=\"$run_dir/user-script.mjs\"", "runner=\"$run_dir/runner.mjs\"", `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$user_script" ${shellQuote(userScriptB64)}`, @@ -5217,6 +5258,104 @@ function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secre ].join("\n"); } +function webProbeScriptRunPathsFromStderr(stderr: string): { runDir: string | null; reportPath: string | null } { + const runDir = lineValueFromText(stderr, "UNIDESK_WEB_PROBE_RUN_DIR"); + const markedReportPath = lineValueFromText(stderr, "UNIDESK_WEB_PROBE_REPORT_PATH"); + const reportPath = markedReportPath ?? (runDir === null ? null : `${runDir.replace(/\/$/u, "")}/web-probe-script-report.json`); + return { + runDir: isSafeWebProbeScriptRunDir(runDir) ? runDir : null, + reportPath: isSafeWebProbeScriptReportPath(reportPath) ? reportPath : null, + }; +} + +function lineValueFromText(text: string, name: string): string | null { + const pattern = new RegExp(`^${name}=([^\\r\\n]+)$`, "mu"); + const match = text.match(pattern); + return match ? match[1].trim() : null; +} + +function readNodeWebProbeScriptReport( + options: NodeWebProbeScriptOptions, + spec: HwlabRuntimeLaneSpec, + reportPath: string | null, +): { source: string; report: Record | null; result: CommandResult | null; degradedReason: string | null; path: string | null } | null { + if (!reportPath) return null; + if (!isSafeWebProbeScriptReportPath(reportPath)) return { source: "unsafe-path", report: null, result: null, degradedReason: "web-probe-report-path-invalid", path: reportPath }; + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, `test -f ${shellQuote(reportPath)} && cat ${shellQuote(reportPath)}`, 55); + if (result.exitCode !== 0 || result.timedOut) { + return { source: "report-file", report: null, result, degradedReason: result.timedOut ? "web-probe-command-timeout" : "web-probe-report-read-failed", path: reportPath }; + } + const report = parseJsonObject(result.stdout); + return { + source: "report-file", + report, + result, + degradedReason: report === null ? "web-probe-report-parse-failed" : null, + path: reportPath, + }; +} + +function readNodeWebProbeScriptArtifacts( + options: NodeWebProbeScriptOptions, + spec: HwlabRuntimeLaneSpec, + runDir: string | null, +): { source: string; artifacts: Record | null; screenshots: Record[]; result: CommandResult | null; degradedReason: string | null } | null { + if (!runDir) return null; + if (!isSafeWebProbeScriptRunDir(runDir)) return { source: "unsafe-path", artifacts: null, screenshots: [], result: null, degradedReason: "web-probe-run-dir-invalid" }; + const script = [ + "set -eu", + `test -d ${shellQuote(runDir)}`, + `find ${shellQuote(runDir)} -maxdepth 1 -type f \\( -name '*.png' -o -name '*.json' \\) -printf '%p\\t%s\\n' | tail -n 30`, + ].join("\n"); + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, 55); + if (result.exitCode !== 0 || result.timedOut) { + return { source: "run-dir", artifacts: null, screenshots: [], result, degradedReason: result.timedOut ? "web-probe-command-timeout" : "web-probe-artifact-list-failed" }; + } + const items = result.stdout.split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [path, sizeText] = line.split("\t"); + const byteCount = Number(sizeText); + return { + kind: path.endsWith(".png") ? "screenshot" : "json", + path, + byteCount: Number.isFinite(byteCount) ? byteCount : null, + }; + }) + .filter((item) => isSafeWebProbeScriptArtifactPath(String(item.path))); + const screenshots = items.filter((item) => item.kind === "screenshot"); + return { + source: "run-dir", + artifacts: { runDir, count: items.length, items }, + screenshots, + result, + degradedReason: null, + }; +} + +function isSafeWebProbeScriptRunDir(value: string | null): value is string { + return typeof value === "string" + && value.length > 0 + && !value.includes("\0") + && !value.includes("..") + && /\.state\/web-probe-script\/run\.[A-Za-z0-9]+$/u.test(value); +} + +function isSafeWebProbeScriptReportPath(value: string | null): value is string { + return typeof value === "string" + && isSafeWebProbeScriptArtifactPath(value) + && value.endsWith("/web-probe-script-report.json"); +} + +function isSafeWebProbeScriptArtifactPath(value: string): boolean { + return typeof value === "string" + && value.length > 0 + && !value.includes("\0") + && !value.includes("..") + && /\.state\/web-probe-script\/run\.[A-Za-z0-9]+\/[^/]+[.](?:png|json)$/u.test(value); +} + function parseSecretOptions(args: string[]): NodeSecretOptions { diff --git a/scripts/src/hwlab-node-web-probe-runner-source.ts b/scripts/src/hwlab-node-web-probe-runner-source.ts index 6603febe..6522e5a0 100644 --- a/scripts/src/hwlab-node-web-probe-runner-source.ts +++ b/scripts/src/hwlab-node-web-probe-runner-source.ts @@ -326,6 +326,9 @@ function scriptHelpers() { goto, gotoRaw, gotoStable, + reloadStable, + safeReload, + gotoCurrentStable, waitWorkbenchReady, waitForReady, fetchJson, @@ -819,6 +822,155 @@ async function gotoRaw(target = "/workbench", options = {}) { return page.url(); } +async function reloadStable(options = {}) { + return stableCurrentPageNavigation("reload-stable", null, normalizeHelperOptions(options)); +} + +async function safeReload(options = {}) { + try { + return await reloadStable(options); + } catch (error) { + const failure = classifiedProbeError(error); + return { + ok: false, + failureKind: failure.failureKind, + error: failure.code, + errorMessage: sanitize(failure.message), + guidance: failure.guidance, + beforeUrl: error && typeof error === "object" && typeof error.beforeUrl === "string" ? error.beforeUrl : null, + finalUrl: currentPageUrl(), + readiness: publicReadiness({ error: failure.code, failureKind: failure.failureKind }), + }; + } +} + +async function gotoCurrentStable(options = {}) { + options = normalizeHelperOptions(options); + const target = typeof options.url === "string" && options.url.length > 0 ? new URL(options.url, baseUrl).toString() : currentPageUrl(); + if (!target) throw stableProbeError("browser-load-jitter", "cannot retry current page navigation because the browser page has no URL", publicReadiness({ error: "browser-page-missing" })); + return stableCurrentPageNavigation("goto-current-stable", target, options); +} + +async function stableCurrentPageNavigation(operation, targetUrl, options = {}) { + options = normalizeHelperOptions(options); + const attempts = boundedInteger(options.attempts, 3, 1, 5); + const retryDelayMs = boundedInteger(options.retryDelayMs, 350, 0, 5000); + const beforeUrl = currentPageUrl(); + let retryUrl = targetUrl; + let lastRecord = null; + + for (let attempt = 1; attempt <= attempts; attempt += 1) { + if (!page || page.isClosed()) page = await context.newPage(); + const attemptPage = page; + const attemptBeforeUrl = currentPageUrl(); + if (!retryUrl && !attemptBeforeUrl) { + throw stableProbeError("browser-load-jitter", "cannot reload because the browser page has no URL", publicReadiness({ error: "browser-page-missing" })); + } + const navigationTarget = retryUrl || attemptBeforeUrl; + const readinessSelectors = normalizeSelectorList(options.selectors, defaultReadinessSelectors(navigationTarget || baseUrl)); + const network = createNetworkTracker(attemptPage, options.apiPattern); + const record = { + ok: false, + attempt, + attempts, + policy: operation, + targetPath: urlPath(navigationTarget), + beforePath: urlPath(attemptBeforeUrl), + finalPath: null, + stage: operation, + error: null, + message: null, + selector: null, + selectors: [], + apiRequestsSent: false, + apiRequestCount: 0, + apiResponseFailed: false, + apiFailureCount: 0, + screenshot: null, + durationMs: 0, + }; + const started = Date.now(); + + try { + if (operation === "reload-stable" && attempt === 1) { + await attemptPage.reload(navigationOptions(options)); + } else { + await attemptPage.goto(navigationTarget, navigationOptions(options)); + } + retryUrl = attemptPage.url(); + record.finalPath = urlPath(attemptPage.url()); + const readiness = await waitForReadyInternal(attemptPage, { + ...options, + selectors: readinessSelectors, + url: attemptPage.url(), + }, network); + applyReadinessToRecord(record, readiness); + if (!readiness.ok) { + record.screenshot = await captureReadinessScreenshot(attemptPage, attempt, options); + lastRecord = record; + } else { + record.ok = true; + record.stage = "ready"; + lastRecord = record; + } + } catch (error) { + record.finalPath = attemptPage && !attemptPage.isClosed() ? urlPath(attemptPage.url()) : null; + record.error = classifyNavigationError(error); + record.message = error instanceof Error ? error.message : String(error); + record.screenshot = await captureReadinessScreenshot(attemptPage, attempt, options); + lastRecord = record; + } finally { + Object.assign(record, network.summary()); + record.durationMs = Date.now() - started; + network.dispose(); + readinessRecords.push(record); + } + + if (record.ok) { + const result = { + ok: true, + operation, + attempt, + attempts, + beforeUrl, + finalUrl: page.url(), + readiness: record, + summary: publicReadiness(), + }; + recordStep(typeof options.name === "string" ? options.name : operation, { + ok: true, + operation, + attempt, + attempts, + beforeUrl, + finalUrl: result.finalUrl, + }); + Object.defineProperty(result, "page", { + enumerable: false, + value: page, + }); + return result; + } + if (attempt < attempts && retryDelayMs > 0) await sleep(retryDelayMs); + } + + const code = lastRecord?.error || "browser-load-jitter"; + const message = lastRecord?.message || operation + " did not become ready"; + recordStep(typeof options.name === "string" ? options.name : operation, { + ok: false, + operation, + attempts, + beforeUrl, + finalUrl: currentPageUrl(), + error: code, + message, + screenshot: lastRecord?.screenshot ?? null, + }); + const error = stableProbeError(code, message, publicReadiness({ error: code })); + error.beforeUrl = beforeUrl; + throw error; +} + async function gotoStable(target = "/workbench", options = {}) { options = normalizeHelperOptions(options); const url = new URL(target, baseUrl).toString(); @@ -1126,13 +1278,16 @@ function navigationOptions(options = {}) { "expectApiRequest", "failOnApiResponseFailed", "freshPage", + "name", "readinessTimeoutMs", + "record", "retryDelayMs", "reusePage", "screenshotOnFailure", "selectorState", "selectors", "stable", + "url", ]) { delete next[key]; } @@ -1300,7 +1455,8 @@ function probeFailureKind(code, message) { if (code === "script-api-misuse") return "script-api-misuse"; if (code === "auth-login-failed") return "auth-failed"; if (code === "browser-failed") return "browser-failed"; - if (code === "browser-load-jitter" || code === "selector-timeout" || code === "api-not-sent" || code === "api-response-failed") return "navigation-failed"; + if (code === "browser-load-jitter") return "browser-navigation-jitter"; + if (code === "selector-timeout" || code === "api-not-sent" || code === "api-response-failed") return "navigation-failed"; if (/browser has been closed|browser executable|failed to launch/iu.test(message)) return "browser-failed"; return "assertion-failed"; } @@ -1310,6 +1466,7 @@ function probeFailureGuidance(failureKind) { if (failureKind === "auth-redirect-login") return "Inspect auth/session bootstrap and finalUrl; the browser returned to /login after CLI-managed auth."; if (failureKind === "same-origin-api-fetch-failed") return "Inspect the failed same-origin API path, current URL, auth state, and browser console/network evidence in the full report."; if (failureKind === "script-api-misuse") return evaluateSingleArgGuidance(); + if (failureKind === "browser-navigation-jitter") return "Use reloadStable(), gotoCurrentStable(), or safeReload() to retry the same browser navigation; inspect readiness.last, lastUrl, and screenshots before treating this as an API fetch failure."; if (failureKind === "navigation-failed") return "Inspect probe.readiness for selector/API readiness details and lastScreenshot for the browser state."; if (failureKind === "auth-failed") return "Inspect probe.auth sourceRef/fingerprint/status; do not print or copy the web login secret."; if (failureKind === "browser-failed") return "Inspect Playwright/browser-launcher availability on the target workspace."; @@ -1765,6 +1922,7 @@ function compactSummaryForStdout(summary) { nextAction: typeof value.nextAction === "string" ? compactText(value.nextAction, 1200) : value.nextAction ?? null, baseUrl: value.baseUrl ?? null, finalUrl: value.finalUrl ?? null, + lastUrl: value.lastUrl ?? null, scriptSha256: value.scriptSha256 ?? null, runDir: value.runDir ?? runDir, reportPath: value.reportPath ?? null, @@ -1869,6 +2027,7 @@ function scriptIssueSummary(payload) { nextAction: ok ? null : issueNextAction(failureKind, payload), baseUrl: payload?.baseUrl ?? null, finalUrl: payload?.finalUrl ?? payload?.lastUrl ?? null, + lastUrl: payload?.lastUrl ?? payload?.finalUrl ?? null, scriptSha256: payload?.scriptSha256 ?? null, runDir, reportPath: payload?.reportPath ?? null, @@ -1897,6 +2056,7 @@ function classifyIssueFailureKind(kind, message = "") { if (/composer-disabled/iu.test(text)) return "composer-disabled"; if (/session-not-selected|session_required/iu.test(text)) return "session-not-selected"; if (/same-origin-api-fetch-failed/iu.test(text)) return "same-origin-api-fetch-failed"; + if (/browser-navigation-jitter|browser-load-jitter|page\.(?:reload|goto)|gotoCurrentStable|reloadStable|safeReload|ERR_NETWORK_CHANGED|ERR_ABORTED|net::|navigation failed|domcontentloaded|chrome-error:\/\/chromewebdata/iu.test(text)) return "browser-navigation-jitter"; if (/api|fetch|network|Failed to fetch|HTTP|status/iu.test(text)) return "network-or-api-fetch-bug"; if (/script-api-misuse|page\.evaluate|safeEvaluate|Too many arguments/iu.test(text)) return "script-bug"; if (/auth|login|credential/iu.test(text)) return "target-auth-bug"; @@ -1910,6 +2070,7 @@ function issueNextAction(failureKind, payload) { if (failureKind === "auth-redirect-login") return "Inspect summary.finalUrl, auth/session bootstrap, and target login redirect handling before rerunning fresh-session."; if (failureKind === "composer-disabled" || failureKind === "session-not-selected") return "Inspect readiness.composer disabledReason and workspace/session summary; create or select a submit-capable session, then rerun the same probe."; if (failureKind === "same-origin-api-fetch-failed") return "Inspect summary.apiMatrix failed rows, current URL, auth state, and browser console/network evidence in reportPath."; + if (failureKind === "browser-navigation-jitter") return "Use reloadStable(), gotoCurrentStable(), or safeReload() to retry the same URL/page reload with bounded attempts; inspect summary.finalUrl, summary.lastScreenshot, and readiness attempts instead of the API matrix first."; if (failureKind === "network-or-api-fetch-bug") return "Inspect summary.apiMatrix failed rows and retry the same node/lane after checking API availability."; if (failureKind === "script-bug") return "Fix the probe script or use safeEvaluate/safeFetchJson/fetchApiMatrix helpers, then rerun the same command."; if (failureKind === "target-auth-bug") return "Inspect credential sourceRef/fingerprint and target /auth/login state; do not print secrets."; diff --git a/scripts/src/hwlab-node-web-probe-summary.ts b/scripts/src/hwlab-node-web-probe-summary.ts index 855d9954..64b98ac9 100644 --- a/scripts/src/hwlab-node-web-probe-summary.ts +++ b/scripts/src/hwlab-node-web-probe-summary.ts @@ -70,6 +70,7 @@ function compactIssueSummary(value: Record): Record