fix: improve web-probe evidence summaries (#466)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
+517
-15
@@ -285,14 +285,16 @@ export function hwlabNodeWebProbeHelp(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --wait-messages-ms 1000",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --url https://hwlab.pikapython.com --fresh-session --message 'ping'",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 --script-file .state/probes/workbench.mjs",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 <<'JS'\nexport default async ({ waitWorkbenchReady, fetchJson, collectText, safeEvaluate, screenshot }) => {\n const ready = await waitWorkbenchReady();\n const workspace = await fetchJson('/v1/workbench/workspace?projectId=prj_hwpod_workbench');\n const workspaceText = await collectText('#workspace');\n const evaluated = await safeEvaluate(({ a, b }) => ({ sum: a + b }), { a: 1, b: 2 });\n await screenshot('workbench.png');\n return { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, workspaceText, evaluated };\n};\nJS",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 <<'JS'\nexport default async ({ waitWorkbenchReady, fetchJson, fetchApiMatrix, recordStep, collectText, safeEvaluate, screenshot }) => {\n const ready = await waitWorkbenchReady();\n const workspace = await fetchJson('/v1/workbench/workspace?projectId=prj_hwpod_workbench');\n const apiMatrix = await fetchApiMatrix(['/v1/workbench/workspace?projectId=prj_hwpod_workbench', '/auth/session']);\n const workspaceText = await collectText('#workspace');\n const evaluated = await safeEvaluate(({ a, b }) => ({ sum: a + b }), { a: 1, b: 2 });\n await screenshot('workbench.png');\n recordStep('workbench-summary', { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, apiMatrixOk: apiMatrix.ok });\n return { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, workspaceText, evaluated };\n};\nJS",
|
||||
],
|
||||
actions: {
|
||||
run: "Run the repo-owned scripts/web-live-dom-probe.mjs helper.",
|
||||
script: "Run caller-provided Playwright JS after CLI-managed /auth/login; the script receives authenticated browser/context/page plus fetchJson/collectText/safeEvaluate/waitWorkbenchReady/screenshotOnError/summarizeWorkspace/summarizeConversation helpers and must not handle secrets itself.",
|
||||
script: "Run caller-provided Playwright JS after CLI-managed /auth/login; the script receives authenticated browser/context/page plus fetchJson/safeFetchJson/fetchApiMatrix/recordStep/collectText/safeEvaluate/waitWorkbenchReady/screenshotOnError/summarizeWorkspace/summarizeConversation helpers and must not handle secrets itself.",
|
||||
},
|
||||
notes: [
|
||||
"Prefer --script-file for reusable probes; stdin heredocs remain supported for one-off probes.",
|
||||
"Issue-ready summary is available under probe.summary; full script report is persisted under probe.reportPath with a SHA-256 fingerprint.",
|
||||
"Use recordStep(name, data) or fetchApiMatrix(paths) to keep structured partial evidence when a later step fails.",
|
||||
"Playwright page.evaluate accepts one serializable argument; use page.evaluate(({ a, b }) => ..., { a, b }) or safeEvaluate(fn, { a, b }).",
|
||||
"Failures include failureKind, errorMessage, scriptSha256, runDir, lastUrl, and lastScreenshot when a screenshot can be captured.",
|
||||
],
|
||||
@@ -4957,8 +4959,12 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||||
const waitAfterSubmitMs = positiveIntegerOption(args, "--wait-after-submit-ms", 1500, 60000);
|
||||
const waitMessagesMs = positiveIntegerOption(args, "--wait-messages-ms", 2500, 60000);
|
||||
const waitAgentTerminalMs = positiveIntegerOption(args, "--wait-agent-terminal-ms", 0, 600000);
|
||||
const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", 0, 200);
|
||||
const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", 0, 60000);
|
||||
const message = optionValue(args, "--message") ?? null;
|
||||
const hasMessage = message !== null;
|
||||
const traceSampleCountDefault = hasMessage ? 2 : 0;
|
||||
const traceSampleIntervalDefault = hasMessage ? 1500 : 0;
|
||||
const traceSampleCount = positiveIntegerOption(args, "--trace-sample-count", traceSampleCountDefault, 200);
|
||||
const traceSampleIntervalMs = positiveIntegerOption(args, "--trace-sample-interval-ms", traceSampleIntervalDefault, 60000);
|
||||
const commandTimeoutAutoSeconds = nodeWebProbeAutoCommandTimeoutSeconds({
|
||||
timeoutMs,
|
||||
waitAfterSubmitMs,
|
||||
@@ -4967,7 +4973,7 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||||
traceSampleCount,
|
||||
traceSampleIntervalMs,
|
||||
freshSession: args.includes("--fresh-session"),
|
||||
hasMessage: optionValue(args, "--message") !== undefined,
|
||||
hasMessage,
|
||||
});
|
||||
const commandTimeoutRaw = optionValue(args, "--command-timeout-seconds");
|
||||
const commandTimeoutUserProvided = commandTimeoutRaw !== undefined;
|
||||
@@ -4985,7 +4991,7 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||||
waitAgentTerminalMs,
|
||||
traceSampleCount,
|
||||
traceSampleIntervalMs,
|
||||
message: optionValue(args, "--message") ?? null,
|
||||
message,
|
||||
conversationId: optionValue(args, "--conversation-id") ?? null,
|
||||
freshSession: args.includes("--fresh-session"),
|
||||
cancelRunning: !args.includes("--no-cancel-running"),
|
||||
@@ -5060,6 +5066,7 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const probe = compactWebProbeResult(parseJsonObject(result.stdout));
|
||||
const passed = result.exitCode === 0 && probe?.status === "pass";
|
||||
const summary = nullableRecord(probe?.summary);
|
||||
const degradedReason = result.timedOut
|
||||
? "web-probe-command-timeout"
|
||||
: typeof probe?.degradedReason === "string"
|
||||
@@ -5081,6 +5088,8 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
|
||||
timedOut: result.timedOut,
|
||||
},
|
||||
degradedReason,
|
||||
failureKind: typeof summary?.failureKind === "string" ? summary.failureKind : null,
|
||||
summary,
|
||||
probe,
|
||||
result: compactCommandResult(result),
|
||||
valuesRedacted: true,
|
||||
@@ -5143,6 +5152,7 @@ function runNodeWebProbeAsync(
|
||||
const report = record(record(poll.status).report);
|
||||
const probe = compactWebProbeResult(Object.keys(report).length > 0 ? report : null);
|
||||
const passed = probe?.status === "pass";
|
||||
const summary = nullableRecord(probe?.summary);
|
||||
const degradedReason = poll.timedOut
|
||||
? "web-probe-command-timeout"
|
||||
: typeof probe?.degradedReason === "string"
|
||||
@@ -5162,6 +5172,8 @@ function runNodeWebProbeAsync(
|
||||
mode: "async",
|
||||
commandTimeout: webProbeCommandTimeoutSummary(options, poll.timedOut),
|
||||
degradedReason,
|
||||
failureKind: typeof summary?.failureKind === "string" ? summary.failureKind : null,
|
||||
summary,
|
||||
job: {
|
||||
jobId,
|
||||
startedAt: start.startedAt ?? null,
|
||||
@@ -5244,6 +5256,14 @@ function runNodeWebProbeScript(
|
||||
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
|
||||
const report = compactWebProbeScriptResult(parseJsonObject(result.stdout));
|
||||
const passed = result.exitCode === 0 && report?.ok === true;
|
||||
const summary = nullableRecord(report?.summary);
|
||||
const degradedReason = typeof summary?.degradedReason === "string"
|
||||
? summary.degradedReason
|
||||
: typeof report?.failureKind === "string"
|
||||
? report.failureKind
|
||||
: result.timedOut
|
||||
? "web-probe-command-timeout"
|
||||
: null;
|
||||
return {
|
||||
ok: passed,
|
||||
status: passed ? "pass" : "blocked",
|
||||
@@ -5254,6 +5274,9 @@ function runNodeWebProbeScript(
|
||||
url: options.url,
|
||||
credential,
|
||||
scriptSource: options.scriptSource,
|
||||
degradedReason,
|
||||
failureKind: typeof summary?.failureKind === "string" ? summary.failureKind : typeof report?.failureKind === "string" ? report.failureKind : null,
|
||||
summary,
|
||||
probe: report,
|
||||
result: compactCommandResultRedacted(result, [material.password ?? ""]),
|
||||
valuesRedacted: true,
|
||||
@@ -5293,7 +5316,8 @@ import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const startedAt = new Date().toISOString();
|
||||
const startedAtMs = Date.now();
|
||||
const startedAt = new Date(startedAtMs).toISOString();
|
||||
const baseUrl = normalizeBaseUrl(process.env.HWLAB_WEB_BASE_URL);
|
||||
const username = process.env.HWLAB_WEB_USER || "admin";
|
||||
const password = process.env.HWLAB_WEB_PASS || "";
|
||||
@@ -5303,6 +5327,7 @@ const timeoutMs = positiveInteger(process.env.UNIDESK_WEB_PROBE_TIMEOUT_MS, 3000
|
||||
const viewport = parseViewport(process.env.UNIDESK_WEB_PROBE_VIEWPORT || "1440x900");
|
||||
const artifactRecords = [];
|
||||
const readinessRecords = [];
|
||||
const stepRecords = [];
|
||||
let browser;
|
||||
let context;
|
||||
let page;
|
||||
@@ -5343,7 +5368,8 @@ try {
|
||||
scriptSha256: userScriptSha256,
|
||||
runDir,
|
||||
auth: publicAuth(auth),
|
||||
script: { ok: scriptOk, result: safeResult },
|
||||
script: { ok: scriptOk, result: safeResult, steps: publicSteps() },
|
||||
steps: publicSteps(),
|
||||
failureKind: failure ? failure.failureKind : null,
|
||||
error: failure ? failure.code : null,
|
||||
errorMessage: failure ? sanitize(failure.message) : null,
|
||||
@@ -5370,6 +5396,8 @@ try {
|
||||
scriptSha256: userScriptSha256,
|
||||
runDir,
|
||||
auth: auth === null ? null : publicAuth(auth),
|
||||
script: { ok: false, steps: publicSteps() },
|
||||
steps: publicSteps(),
|
||||
failureKind: failure.failureKind,
|
||||
error: failure.code,
|
||||
errorMessage: sanitize(failure.message),
|
||||
@@ -5608,6 +5636,9 @@ function scriptHelpers() {
|
||||
waitWorkbenchReady,
|
||||
waitForReady,
|
||||
fetchJson,
|
||||
safeFetchJson,
|
||||
fetchApiMatrix,
|
||||
recordStep,
|
||||
collectText,
|
||||
safeEvaluate,
|
||||
summarizeWorkspace,
|
||||
@@ -5622,6 +5653,99 @@ function scriptHelpers() {
|
||||
return helpers;
|
||||
}
|
||||
|
||||
function recordStep(name, data = {}, options = {}) {
|
||||
const normalizedName = String(name || "step").replace(/[^A-Za-z0-9_.:-]/gu, "-").slice(0, 80) || "step";
|
||||
const item = {
|
||||
index: stepRecords.length,
|
||||
name: normalizedName,
|
||||
ok: data && typeof data === "object" && typeof data.ok === "boolean" ? data.ok : null,
|
||||
atMs: Date.now() - startedAtMs,
|
||||
data: sanitize(data),
|
||||
};
|
||||
stepRecords.push(item);
|
||||
while (stepRecords.length > boundedInteger(options.maxSteps, 80, 1, 200)) stepRecords.shift();
|
||||
return deepClonePlain(item);
|
||||
}
|
||||
|
||||
function publicSteps() {
|
||||
return stepRecords.slice(-50).map((item) => deepClonePlain(item));
|
||||
}
|
||||
|
||||
async function safeFetchJson(target, options = {}) {
|
||||
try {
|
||||
return await fetchJson(target, { ...normalizeHelperOptions(options), throwOnError: false });
|
||||
} catch (error) {
|
||||
const failure = classifiedProbeError(error);
|
||||
return {
|
||||
ok: false,
|
||||
path: typeof target === "string" ? urlPath(target) : String(target ?? ""),
|
||||
status: 0,
|
||||
statusText: "helper-error",
|
||||
failureKind: failure.failureKind,
|
||||
error: sanitize(failure.message),
|
||||
guidance: failure.guidance,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchApiMatrix(paths, options = {}) {
|
||||
options = normalizeHelperOptions(options);
|
||||
const items = normalizeApiMatrixItems(paths);
|
||||
const rows = [];
|
||||
for (const item of items) {
|
||||
const response = await safeFetchJson(item.path, {
|
||||
headers: item.headers ?? options.headers,
|
||||
method: item.method ?? options.method,
|
||||
body: item.body ?? undefined,
|
||||
});
|
||||
rows.push(compactApiMatrixResponse(item, response));
|
||||
}
|
||||
const failed = rows.filter((row) => row.ok !== true);
|
||||
const matrix = {
|
||||
ok: failed.length === 0,
|
||||
count: rows.length,
|
||||
okCount: rows.length - failed.length,
|
||||
failedCount: failed.length,
|
||||
items: rows,
|
||||
};
|
||||
if (options.record !== false) recordStep(typeof options.name === "string" ? options.name : "api-matrix", matrix);
|
||||
return matrix;
|
||||
}
|
||||
|
||||
function normalizeApiMatrixItems(value) {
|
||||
const list = Array.isArray(value) ? value : [value];
|
||||
return list.slice(0, 20).map((item, index) => {
|
||||
if (typeof item === "string") return { name: item, path: item, method: "GET" };
|
||||
if (item && typeof item === "object") {
|
||||
const path = typeof item.path === "string" ? item.path : typeof item.url === "string" ? item.url : "/";
|
||||
return {
|
||||
name: typeof item.name === "string" ? item.name : path,
|
||||
path,
|
||||
method: typeof item.method === "string" ? item.method : "GET",
|
||||
headers: item.headers,
|
||||
body: item.body,
|
||||
};
|
||||
}
|
||||
return { name: "item-" + index, path: "/", method: "GET" };
|
||||
});
|
||||
}
|
||||
|
||||
function compactApiMatrixResponse(item, response) {
|
||||
const body = response && typeof response.body === "object" && response.body !== null ? response.body : null;
|
||||
return {
|
||||
name: item.name,
|
||||
path: response?.path ?? item.path,
|
||||
method: item.method ?? "GET",
|
||||
ok: response?.ok === true,
|
||||
status: response?.status ?? null,
|
||||
statusText: response?.statusText ?? null,
|
||||
error: response?.error ?? response?.parseError ?? null,
|
||||
failureKind: response?.failureKind ?? null,
|
||||
bodyKeys: body && !Array.isArray(body) ? Object.keys(body).slice(0, 20) : [],
|
||||
bodyPreview: body ? normalizeTextPreview(JSON.stringify(sanitize(body)), 240) : response?.textPreview ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
async function safeEvaluate(fn, arg = undefined, options = {}) {
|
||||
options = normalizeHelperOptions(options);
|
||||
if (typeof fn !== "function") {
|
||||
@@ -6651,23 +6775,125 @@ function redactString(text) {
|
||||
}
|
||||
|
||||
async function emit(payload) {
|
||||
process.stdout.write(JSON.stringify(sanitize(payload), null, 2) + "\n");
|
||||
const enriched = {
|
||||
...payload,
|
||||
steps: publicSteps(),
|
||||
artifacts: { runDir, items: artifactRecords },
|
||||
};
|
||||
enriched.summary = scriptIssueSummary(enriched);
|
||||
const reportFile = artifactPath("web-probe-script-report.json");
|
||||
let reportArtifact = null;
|
||||
try {
|
||||
await writeFile(reportFile, JSON.stringify(sanitize(enriched), null, 2) + "\n", "utf8");
|
||||
const fileStat = await stat(reportFile);
|
||||
reportArtifact = {
|
||||
kind: "json",
|
||||
path: reportFile,
|
||||
byteCount: fileStat.size,
|
||||
sha256: await sha256File(reportFile),
|
||||
};
|
||||
artifactRecords.push({ ...reportArtifact });
|
||||
} catch (error) {
|
||||
reportArtifact = {
|
||||
kind: "json",
|
||||
path: reportFile,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
const finalPayload = {
|
||||
...enriched,
|
||||
reportPath: reportArtifact && typeof reportArtifact.path === "string" ? reportArtifact.path : null,
|
||||
reportSha256: reportArtifact && typeof reportArtifact.sha256 === "string" ? reportArtifact.sha256 : null,
|
||||
artifacts: { runDir, items: artifactRecords },
|
||||
};
|
||||
finalPayload.summary = scriptIssueSummary(finalPayload);
|
||||
process.stdout.write(JSON.stringify(sanitize(finalPayload), null, 2) + "\n");
|
||||
}
|
||||
|
||||
function scriptIssueSummary(payload) {
|
||||
const artifacts = Array.isArray(payload?.artifacts?.items) ? payload.artifacts.items : artifactRecords;
|
||||
const screenshots = artifacts
|
||||
.filter((item) => item && typeof item === "object" && item.kind === "screenshot")
|
||||
.slice(-5)
|
||||
.map((item) => ({
|
||||
path: item.path ?? null,
|
||||
sha256: item.sha256 ?? null,
|
||||
byteCount: item.byteCount ?? null,
|
||||
}));
|
||||
const apiMatrix = latestApiMatrixFromSteps(publicSteps());
|
||||
const ok = payload?.ok === true;
|
||||
const degradedReason = ok ? null : payload?.error ?? payload?.failureKind ?? "web-probe-script-failed";
|
||||
const failureKind = ok ? null : classifyIssueFailureKind(payload?.failureKind ?? payload?.error ?? degradedReason, payload?.errorMessage);
|
||||
return {
|
||||
ok,
|
||||
status: payload?.status ?? null,
|
||||
degradedReason,
|
||||
failureKind,
|
||||
failedCondition: ok ? null : payload?.errorMessage ?? payload?.error ?? payload?.failureKind ?? "script did not pass",
|
||||
nextAction: ok ? null : issueNextAction(failureKind, payload),
|
||||
baseUrl: payload?.baseUrl ?? null,
|
||||
finalUrl: payload?.finalUrl ?? payload?.lastUrl ?? null,
|
||||
scriptSha256: payload?.scriptSha256 ?? null,
|
||||
runDir,
|
||||
reportPath: payload?.reportPath ?? null,
|
||||
reportSha256: payload?.reportSha256 ?? null,
|
||||
lastScreenshot: payload?.lastScreenshot ?? null,
|
||||
screenshots,
|
||||
apiMatrix,
|
||||
stepCount: stepRecords.length,
|
||||
lastStep: stepRecords.length > 0 ? deepClonePlain(stepRecords[stepRecords.length - 1]) : null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function latestApiMatrixFromSteps(steps) {
|
||||
for (let index = steps.length - 1; index >= 0; index -= 1) {
|
||||
const step = steps[index];
|
||||
const data = step && typeof step === "object" ? step.data : null;
|
||||
if (data && typeof data === "object" && Array.isArray(data.items) && typeof data.failedCount === "number") return data;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function classifyIssueFailureKind(kind, message = "") {
|
||||
const text = String(kind ?? "") + " " + String(message ?? "");
|
||||
if (/api|fetch|network|Failed to fetch|HTTP|status/iu.test(text)) return "network-or-api-fetch-bug";
|
||||
if (/script-api-misuse|page\.evaluate|safeEvaluate|Too many arguments/iu.test(text)) return "script-bug";
|
||||
if (/auth|login|credential/iu.test(text)) return "target-auth-bug";
|
||||
if (/browser|chromium|playwright|executable/iu.test(text)) return "browser-environment-bug";
|
||||
if (/assert|expect|ok:false|validation|final-response|message/iu.test(text)) return "unmet-expectation";
|
||||
if (/script|ReferenceError|TypeError|SyntaxError/iu.test(text)) return "script-bug";
|
||||
return "user-facing-web-bug";
|
||||
}
|
||||
|
||||
function issueNextAction(failureKind, payload) {
|
||||
if (failureKind === "network-or-api-fetch-bug") return "Inspect summary.apiMatrix failed rows and retry the same node/lane after checking API availability.";
|
||||
if (failureKind === "script-bug") return "Fix the probe script or use safeEvaluate/safeFetchJson/fetchApiMatrix helpers, then rerun the same command.";
|
||||
if (failureKind === "target-auth-bug") return "Inspect credential sourceRef/fingerprint and target /auth/login state; do not print secrets.";
|
||||
if (failureKind === "browser-environment-bug") return "Inspect Playwright/browser-launcher availability in the target workspace.";
|
||||
if (failureKind === "unmet-expectation") return "Inspect summary.failedCondition, DOM/API steps, and screenshots to decide whether the Web behavior or the assertion is stale.";
|
||||
return payload?.guidance ?? "Inspect reportPath for full redacted details and rerun the same node/lane entry after the root cause is fixed.";
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
function compactWebProbeScriptResult(report: Record<string, unknown> | null): Record<string, unknown> | null {
|
||||
if (report === null) return null;
|
||||
const summary = compactIssueSummary(record(report.summary));
|
||||
return {
|
||||
ok: report.ok === true,
|
||||
status: typeof report.status === "string" ? report.status : null,
|
||||
summary,
|
||||
baseUrl: typeof report.baseUrl === "string" ? report.baseUrl : null,
|
||||
finalUrl: typeof report.finalUrl === "string" ? report.finalUrl : null,
|
||||
lastUrl: typeof report.lastUrl === "string" ? report.lastUrl : null,
|
||||
scriptSha256: typeof report.scriptSha256 === "string" ? report.scriptSha256 : null,
|
||||
runDir: typeof report.runDir === "string" ? report.runDir : null,
|
||||
reportPath: typeof report.reportPath === "string" ? report.reportPath : null,
|
||||
reportSha256: typeof report.reportSha256 === "string" ? report.reportSha256 : null,
|
||||
auth: record(report.auth),
|
||||
script: record(report.script),
|
||||
script: compactWebProbeScriptBlock(report.script),
|
||||
steps: compactWebProbeSteps(report.steps),
|
||||
failureKind: typeof report.failureKind === "string" ? report.failureKind : null,
|
||||
guidance: typeof report.guidance === "string" ? report.guidance : null,
|
||||
lastScreenshot: nullableRecord(report.lastScreenshot),
|
||||
@@ -6679,6 +6905,113 @@ function compactWebProbeScriptResult(report: Record<string, unknown> | null): Re
|
||||
};
|
||||
}
|
||||
|
||||
function compactWebProbeScriptBlock(value: unknown): Record<string, unknown> {
|
||||
const script = record(value);
|
||||
return {
|
||||
ok: script.ok === true,
|
||||
result: compactJsonForIssue(script.result),
|
||||
stepCount: Array.isArray(script.steps) ? script.steps.length : null,
|
||||
};
|
||||
}
|
||||
|
||||
function compactWebProbeSteps(value: unknown): Record<string, unknown>[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.slice(-30).map((item) => {
|
||||
const step = record(item);
|
||||
return {
|
||||
index: typeof step.index === "number" ? step.index : null,
|
||||
name: typeof step.name === "string" ? step.name : null,
|
||||
ok: typeof step.ok === "boolean" ? step.ok : null,
|
||||
atMs: typeof step.atMs === "number" ? step.atMs : null,
|
||||
data: compactStepDataForIssue(step.data),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function compactIssueSummary(value: Record<string, unknown>): Record<string, unknown> {
|
||||
if (Object.keys(value).length === 0) return {};
|
||||
return {
|
||||
ok: value.ok === true,
|
||||
status: typeof value.status === "string" ? value.status : null,
|
||||
degradedReason: typeof value.degradedReason === "string" ? value.degradedReason : null,
|
||||
failureKind: typeof value.failureKind === "string" ? value.failureKind : null,
|
||||
failedCondition: typeof value.failedCondition === "string" ? value.failedCondition : null,
|
||||
nextAction: typeof value.nextAction === "string" ? value.nextAction : null,
|
||||
finalUrl: typeof value.finalUrl === "string" ? value.finalUrl : null,
|
||||
scriptSha256: typeof value.scriptSha256 === "string" ? value.scriptSha256 : null,
|
||||
runDir: typeof value.runDir === "string" ? value.runDir : null,
|
||||
reportPath: typeof value.reportPath === "string" ? value.reportPath : null,
|
||||
reportSha256: typeof value.reportSha256 === "string" ? value.reportSha256 : null,
|
||||
lastScreenshot: nullableRecord(value.lastScreenshot),
|
||||
screenshots: Array.isArray(value.screenshots) ? value.screenshots.slice(-5).map(nullableRecord).filter((item): item is Record<string, unknown> => item !== null) : [],
|
||||
apiMatrix: compactApiMatrixSummary(value.apiMatrix),
|
||||
stepCount: typeof value.stepCount === "number" ? value.stepCount : null,
|
||||
lastStep: compactStepForIssue(value.lastStep),
|
||||
valuesRedacted: value.valuesRedacted === true,
|
||||
};
|
||||
}
|
||||
|
||||
function compactJsonForIssue(value: unknown, depth = 0): unknown {
|
||||
if (value === null || value === undefined) return value ?? null;
|
||||
if (typeof value === "string") return value.replace(/\s+/gu, " ").trim().slice(0, 600);
|
||||
if (typeof value === "number" || typeof value === "boolean") return value;
|
||||
if (depth >= 5) return "[max-depth]";
|
||||
if (Array.isArray(value)) return value.slice(0, 30).map((item) => compactJsonForIssue(item, depth + 1));
|
||||
if (typeof value === "object") {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, nested] of Object.entries(value as Record<string, unknown>).slice(0, 40)) {
|
||||
out[key] = compactJsonForIssue(nested, depth + 1);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return String(value).slice(0, 600);
|
||||
}
|
||||
|
||||
function compactStepForIssue(value: unknown): Record<string, unknown> | null {
|
||||
const step = nullableRecord(value);
|
||||
if (step === null) return null;
|
||||
return {
|
||||
index: typeof step.index === "number" ? step.index : null,
|
||||
name: typeof step.name === "string" ? step.name : null,
|
||||
ok: typeof step.ok === "boolean" ? step.ok : null,
|
||||
atMs: typeof step.atMs === "number" ? step.atMs : null,
|
||||
data: compactStepDataForIssue(step.data),
|
||||
};
|
||||
}
|
||||
|
||||
function compactStepDataForIssue(value: unknown): unknown {
|
||||
const data = nullableRecord(value);
|
||||
if (data === null) return compactJsonForIssue(value);
|
||||
if (Array.isArray(data.items) && typeof data.failedCount === "number") return compactApiMatrixSummary(data);
|
||||
return compactJsonForIssue(data);
|
||||
}
|
||||
|
||||
function compactApiMatrixSummary(value: unknown): Record<string, unknown> | null {
|
||||
const matrix = nullableRecord(value);
|
||||
if (matrix === null) return null;
|
||||
return {
|
||||
ok: matrix.ok === true,
|
||||
count: typeof matrix.count === "number" ? matrix.count : null,
|
||||
okCount: typeof matrix.okCount === "number" ? matrix.okCount : null,
|
||||
failedCount: typeof matrix.failedCount === "number" ? matrix.failedCount : null,
|
||||
items: Array.isArray(matrix.items)
|
||||
? matrix.items.slice(0, 12).map((item) => {
|
||||
const row = record(item);
|
||||
return {
|
||||
name: typeof row.name === "string" ? row.name : null,
|
||||
path: typeof row.path === "string" ? row.path : null,
|
||||
method: typeof row.method === "string" ? row.method : null,
|
||||
ok: row.ok === true,
|
||||
status: typeof row.status === "number" ? row.status : null,
|
||||
error: typeof row.error === "string" ? row.error : null,
|
||||
failureKind: typeof row.failureKind === "string" ? row.failureKind : null,
|
||||
bodyKeys: Array.isArray(row.bodyKeys) ? row.bodyKeys.filter((key) => typeof key === "string").slice(0, 12) : [],
|
||||
};
|
||||
})
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
function parseSecretOptions(args: string[]): NodeSecretOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") {
|
||||
@@ -9522,18 +9855,26 @@ function parseJsonObject(text: string): Record<string, unknown> | null {
|
||||
function compactWebProbeResult(report: Record<string, unknown> | null): Record<string, unknown> | null {
|
||||
if (report === null) return null;
|
||||
const dom = record(report.dom);
|
||||
const performance = record(report.performance);
|
||||
const trace = record(report.trace);
|
||||
const session = record(report.session);
|
||||
const performance = compactWebProbePerformance(report.performance);
|
||||
const trace = compactWebProbeTrace(report.trace);
|
||||
const session = compactWebProbeSession(report.session);
|
||||
const artifacts = record(report.artifacts);
|
||||
const promptValidation = compactPromptValidation(report.promptValidation);
|
||||
const degradedReason = typeof report.degradedReason === "string" ? report.degradedReason : null;
|
||||
const summary = webProbeRunIssueSummary({ report, session, trace, promptValidation, artifacts, degradedReason });
|
||||
return {
|
||||
ok: report.ok === true,
|
||||
status: typeof report.status === "string" ? report.status : null,
|
||||
summary,
|
||||
finalUrl: typeof report.finalUrl === "string" ? report.finalUrl : null,
|
||||
error: typeof report.error === "string" ? report.error : null,
|
||||
degradedReason: typeof report.degradedReason === "string" ? report.degradedReason : null,
|
||||
degradedReason,
|
||||
failureKind: typeof summary.failureKind === "string" ? summary.failureKind : null,
|
||||
nextAction: typeof summary.nextAction === "string" ? summary.nextAction : null,
|
||||
actions: compactWebProbeActions(report.actions),
|
||||
session,
|
||||
trace,
|
||||
promptValidation,
|
||||
performance,
|
||||
traceSamples: compactWebProbeTraceSamples(report.traceSamples),
|
||||
dom: {
|
||||
@@ -9543,11 +9884,172 @@ function compactWebProbeResult(report: Record<string, unknown> | null): Record<s
|
||||
sessionRail: record(dom.sessionRail),
|
||||
},
|
||||
failureDom: record(report.failureDom),
|
||||
artifacts: record(report.artifacts),
|
||||
artifacts,
|
||||
safety: record(report.safety),
|
||||
};
|
||||
}
|
||||
|
||||
function compactWebProbeSession(value: unknown): Record<string, unknown> {
|
||||
const session = record(value);
|
||||
const freshSession = record(session.freshSession);
|
||||
return {
|
||||
freshSessionRequested: session.freshSessionRequested === true,
|
||||
freshSessionAligned: typeof session.freshSessionAligned === "boolean" ? session.freshSessionAligned : null,
|
||||
conversationId: session.conversationId ?? null,
|
||||
routeConversationId: session.routeConversationId ?? null,
|
||||
selectedConversationId: session.selectedConversationId ?? null,
|
||||
sessionId: session.sessionId ?? null,
|
||||
revision: session.revision ?? null,
|
||||
messageCount: session.messageCount ?? null,
|
||||
degradedReason: session.degradedReason ?? null,
|
||||
freshSession: Object.keys(freshSession).length === 0 ? null : {
|
||||
settled: freshSession.settled === true,
|
||||
aligned: freshSession.aligned === true,
|
||||
reason: freshSession.reason ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function compactWebProbeTrace(value: unknown): Record<string, unknown> {
|
||||
const trace = record(value);
|
||||
const latestFetch = record(trace.latestFetch);
|
||||
return {
|
||||
requested: trace.requested === true,
|
||||
sampleCount: trace.sampleCount ?? null,
|
||||
conversationId: trace.conversationId ?? null,
|
||||
sessionId: trace.sessionId ?? null,
|
||||
traceId: trace.traceId ?? null,
|
||||
finalAgentStatus: trace.finalAgentStatus ?? null,
|
||||
finalTraceStatus: trace.finalTraceStatus ?? null,
|
||||
finalDomRowCount: trace.finalDomRowCount ?? null,
|
||||
firstDomRowVisibleAt: trace.firstDomRowVisibleAt ?? null,
|
||||
firstDomRowPreview: typeof trace.firstDomRowPreview === "string" ? trace.firstDomRowPreview.slice(0, 240) : null,
|
||||
firstRestEventVisibleAt: trace.firstRestEventVisibleAt ?? null,
|
||||
firstRestEventCount: trace.firstRestEventCount ?? null,
|
||||
restTraceOk: trace.restTraceOk === true,
|
||||
failedFetchCount: trace.failedFetchCount ?? null,
|
||||
latestFetch: Object.keys(latestFetch).length === 0 ? null : {
|
||||
ok: latestFetch.ok === true,
|
||||
httpStatus: latestFetch.httpStatus ?? null,
|
||||
eventCount: latestFetch.eventCount ?? null,
|
||||
traceStatus: latestFetch.traceStatus ?? null,
|
||||
degradedReason: latestFetch.degradedReason ?? null,
|
||||
attempts: Array.isArray(latestFetch.attempts) ? latestFetch.attempts.length : null,
|
||||
latestEventPreview: typeof latestFetch.latestEventPreview === "string" ? latestFetch.latestEventPreview.slice(0, 240) : null,
|
||||
},
|
||||
degradedReason: trace.degradedReason ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function compactWebProbePerformance(value: unknown): Record<string, unknown> {
|
||||
const performance = record(value);
|
||||
return {
|
||||
startedAt: performance.startedAt ?? null,
|
||||
submittedAt: performance.submittedAt ?? null,
|
||||
firstDomTraceRowAt: performance.firstDomTraceRowAt ?? null,
|
||||
firstRestTraceEventAt: performance.firstRestTraceEventAt ?? null,
|
||||
submitToFirstDomTraceRowMs: performance.submitToFirstDomTraceRowMs ?? null,
|
||||
submitToFirstRestTraceEventMs: performance.submitToFirstRestTraceEventMs ?? null,
|
||||
totalSampleWindowMs: performance.totalSampleWindowMs ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function compactPromptValidation(value: unknown): Record<string, unknown> {
|
||||
const prompt = record(value);
|
||||
const finalResponse = record(prompt.finalResponse);
|
||||
return {
|
||||
ok: prompt.ok === true,
|
||||
applicable: prompt.applicable === true,
|
||||
reason: typeof prompt.reason === "string" ? prompt.reason : null,
|
||||
failures: Array.isArray(prompt.failures) ? prompt.failures.filter((item) => typeof item === "string").slice(0, 20) : [],
|
||||
finalResponse: Object.keys(finalResponse).length === 0 ? null : {
|
||||
textChars: typeof finalResponse.textChars === "number" ? finalResponse.textChars : null,
|
||||
textPreview: typeof finalResponse.textPreview === "string" ? finalResponse.textPreview.slice(0, 500) : null,
|
||||
markdown: nullableRecord(finalResponse.markdown),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function webProbeRunIssueSummary(input: {
|
||||
report: Record<string, unknown>;
|
||||
session: Record<string, unknown>;
|
||||
trace: Record<string, unknown>;
|
||||
promptValidation: Record<string, unknown>;
|
||||
artifacts: Record<string, unknown>;
|
||||
degradedReason: string | null;
|
||||
}): Record<string, unknown> {
|
||||
const failureKind = webProbeRunFailureKind(input.degradedReason, input.promptValidation);
|
||||
const failedCondition = webProbeRunFailedCondition(input.degradedReason, input.promptValidation);
|
||||
const promptSubmitted = record(input.report.safety).promptSubmitted === true;
|
||||
const traceRequested = input.trace.requested === true || record(input.report.safety).traceSamplingRequested === true;
|
||||
return {
|
||||
ok: input.report.status === "pass" && !input.degradedReason,
|
||||
status: typeof input.report.status === "string" ? input.report.status : null,
|
||||
degradedReason: input.degradedReason,
|
||||
failureKind,
|
||||
failedCondition,
|
||||
nextAction: webProbeRunNextAction(failureKind, input.degradedReason, input.promptValidation, promptSubmitted, traceRequested),
|
||||
finalUrl: typeof input.report.finalUrl === "string" ? input.report.finalUrl : null,
|
||||
conversationId: input.session.conversationId ?? input.trace.conversationId ?? null,
|
||||
sessionId: input.session.sessionId ?? input.trace.sessionId ?? null,
|
||||
traceId: input.trace.traceId ?? null,
|
||||
traceRequested,
|
||||
traceStatus: input.trace.finalTraceStatus ?? null,
|
||||
agentStatus: input.trace.finalAgentStatus ?? null,
|
||||
messageCount: input.session.messageCount ?? record(input.report.dom).messageCount ?? null,
|
||||
promptValidation: input.promptValidation,
|
||||
reportPath: typeof input.artifacts.reportPath === "string" ? input.artifacts.reportPath : null,
|
||||
reportSha256: typeof input.artifacts.reportSha256 === "string" ? input.artifacts.reportSha256 : null,
|
||||
screenshotPath: typeof input.artifacts.screenshotPath === "string" ? input.artifacts.screenshotPath : null,
|
||||
screenshotSha256: typeof input.artifacts.screenshotSha256 === "string" ? input.artifacts.screenshotSha256 : null,
|
||||
traceRecommendation: promptSubmitted && !traceRequested
|
||||
? "rerun with --trace-sample-count 2 --trace-sample-interval-ms 1500, or rely on the new default for prompt probes"
|
||||
: null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function webProbeRunFailureKind(degradedReason: string | null, promptValidation: Record<string, unknown>): string | null {
|
||||
if (degradedReason === null) return null;
|
||||
if (degradedReason === "prompt-validation-failed") return "unmet-expectation";
|
||||
if (/trace-fetch|api|fetch|http|network/iu.test(degradedReason)) return "network-or-api-fetch-bug";
|
||||
if (/auth|login|credential/iu.test(degradedReason)) return "target-auth-bug";
|
||||
if (/browser|timeout|playwright|chromium/iu.test(degradedReason)) return "browser-environment-bug";
|
||||
const failures = Array.isArray(promptValidation.failures) ? promptValidation.failures.join(" ") : "";
|
||||
if (/final-response|agent-message|markdown|completed/iu.test(failures)) return "unmet-expectation";
|
||||
return "user-facing-web-bug";
|
||||
}
|
||||
|
||||
function webProbeRunFailedCondition(degradedReason: string | null, promptValidation: Record<string, unknown>): string | null {
|
||||
if (degradedReason === "prompt-validation-failed") {
|
||||
const failures = Array.isArray(promptValidation.failures) ? promptValidation.failures.filter((item) => typeof item === "string") : [];
|
||||
return failures.length > 0 ? failures.join(",") : "prompt final response validation failed";
|
||||
}
|
||||
return degradedReason;
|
||||
}
|
||||
|
||||
function webProbeRunNextAction(
|
||||
failureKind: string | null,
|
||||
degradedReason: string | null,
|
||||
promptValidation: Record<string, unknown>,
|
||||
promptSubmitted: boolean,
|
||||
traceRequested: boolean,
|
||||
): string | null {
|
||||
if (failureKind === null) return null;
|
||||
if (failureKind === "unmet-expectation") {
|
||||
const failures = Array.isArray(promptValidation.failures) ? promptValidation.failures.join(",") : "";
|
||||
if (/agent-not-completed/iu.test(failures)) return "Increase --wait-agent-terminal-ms or inspect the Code Agent terminal state for the conversation/trace ids in summary.";
|
||||
if (/final-response/iu.test(failures)) return "Inspect Workbench message projection and conversation detail; terminal messages must expose a final response.";
|
||||
if (/markdown/iu.test(failures)) return "Check whether the prompt expectation still requires markdown structure; stale assertions should be removed instead of preserved.";
|
||||
return "Inspect promptValidation.failures, screenshots, and reportPath to decide whether the Web behavior or the assertion is stale.";
|
||||
}
|
||||
if (failureKind === "network-or-api-fetch-bug") return "Inspect trace/session API fetch fields in reportPath and retry after checking target API availability.";
|
||||
if (failureKind === "target-auth-bug") return "Inspect credential sourceRef/fingerprint and /auth/login status; do not print secrets.";
|
||||
if (failureKind === "browser-environment-bug") return "Inspect browser launcher and Playwright availability on the target workspace.";
|
||||
if (promptSubmitted && !traceRequested) return "Rerun with trace sampling if trace evidence is required.";
|
||||
return degradedReason === null ? null : "Inspect reportPath for full redacted details, then rerun the same node/lane entry.";
|
||||
}
|
||||
|
||||
function compactWebProbeTraceSamples(value: unknown): Record<string, unknown>[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.map(record).map((sample) => {
|
||||
|
||||
Reference in New Issue
Block a user