Codex
431 lines
21 KiB
TypeScript
431 lines
21 KiB
TypeScript
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// Responsibility: Redacted web-probe summaries and issue-ready compaction helpers.
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function nullableRecord(value: unknown): Record<string, unknown> | null {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
|
}
|
|
|
|
export function compactWebProbeScriptResult(report: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (report === null) return null;
|
|
const summary = compactIssueSummary(record(report.summary));
|
|
return {
|
|
ok: report.ok === true,
|
|
status: typeof report.status === "string" ? report.status : null,
|
|
summary,
|
|
baseUrl: typeof report.baseUrl === "string" ? report.baseUrl : null,
|
|
finalUrl: typeof report.finalUrl === "string" ? report.finalUrl : null,
|
|
lastUrl: typeof report.lastUrl === "string" ? report.lastUrl : null,
|
|
scriptSha256: typeof report.scriptSha256 === "string" ? report.scriptSha256 : null,
|
|
runDir: typeof report.runDir === "string" ? report.runDir : null,
|
|
reportPath: typeof report.reportPath === "string" ? report.reportPath : null,
|
|
reportSha256: typeof report.reportSha256 === "string" ? report.reportSha256 : null,
|
|
auth: record(report.auth),
|
|
script: compactWebProbeScriptBlock(report.script),
|
|
steps: compactWebProbeSteps(report.steps),
|
|
failureKind: typeof report.failureKind === "string" ? report.failureKind : null,
|
|
guidance: typeof report.guidance === "string" ? report.guidance : null,
|
|
lastScreenshot: nullableRecord(report.lastScreenshot),
|
|
readiness: record(report.readiness),
|
|
artifacts: record(report.artifacts),
|
|
error: typeof report.error === "string" ? report.error : null,
|
|
errorMessage: typeof report.errorMessage === "string" ? report.errorMessage : null,
|
|
safety: record(report.safety),
|
|
};
|
|
}
|
|
|
|
function compactWebProbeScriptBlock(value: unknown): Record<string, unknown> {
|
|
const script = record(value);
|
|
return {
|
|
ok: script.ok === true,
|
|
result: compactJsonForIssue(script.result),
|
|
stepCount: Array.isArray(script.steps) ? script.steps.length : null,
|
|
};
|
|
}
|
|
|
|
function compactWebProbeSteps(value: unknown): Record<string, unknown>[] {
|
|
if (!Array.isArray(value)) return [];
|
|
return value.slice(-5).map((item) => {
|
|
const step = record(item);
|
|
return {
|
|
index: typeof step.index === "number" ? step.index : null,
|
|
name: typeof step.name === "string" ? step.name : null,
|
|
ok: typeof step.ok === "boolean" ? step.ok : null,
|
|
atMs: typeof step.atMs === "number" ? step.atMs : null,
|
|
data: compactStepDataForIssue(step.data),
|
|
};
|
|
});
|
|
}
|
|
|
|
function compactIssueSummary(value: Record<string, unknown>): Record<string, unknown> {
|
|
if (Object.keys(value).length === 0) return {};
|
|
return {
|
|
ok: value.ok === true,
|
|
status: typeof value.status === "string" ? value.status : null,
|
|
degradedReason: typeof value.degradedReason === "string" ? value.degradedReason : null,
|
|
failureKind: typeof value.failureKind === "string" ? value.failureKind : null,
|
|
failedCondition: typeof value.failedCondition === "string" ? value.failedCondition : null,
|
|
nextAction: typeof value.nextAction === "string" ? value.nextAction : null,
|
|
finalUrl: typeof value.finalUrl === "string" ? value.finalUrl : null,
|
|
lastUrl: typeof value.lastUrl === "string" ? value.lastUrl : null,
|
|
scriptSha256: typeof value.scriptSha256 === "string" ? value.scriptSha256 : null,
|
|
runDir: typeof value.runDir === "string" ? value.runDir : null,
|
|
reportPath: typeof value.reportPath === "string" ? value.reportPath : null,
|
|
reportSha256: typeof value.reportSha256 === "string" ? value.reportSha256 : null,
|
|
lastScreenshot: nullableRecord(value.lastScreenshot),
|
|
screenshots: Array.isArray(value.screenshots) ? value.screenshots.slice(-5).map(nullableRecord).filter((item): item is Record<string, unknown> => item !== null) : [],
|
|
apiMatrix: compactApiMatrixSummary(value.apiMatrix),
|
|
stepCount: typeof value.stepCount === "number" ? value.stepCount : null,
|
|
lastStep: compactStepForIssue(value.lastStep),
|
|
valuesRedacted: value.valuesRedacted === true,
|
|
};
|
|
}
|
|
|
|
function compactJsonForIssue(value: unknown, depth = 0): unknown {
|
|
if (value === null || value === undefined) return value ?? null;
|
|
if (typeof value === "string") return value.replace(/\s+/gu, " ").trim().slice(0, 240);
|
|
if (typeof value === "number" || typeof value === "boolean") return value;
|
|
if (depth >= 4) return "[max-depth]";
|
|
if (Array.isArray(value)) return value.slice(0, 8).map((item) => compactJsonForIssue(item, depth + 1));
|
|
if (typeof value === "object") {
|
|
const out: Record<string, unknown> = {};
|
|
for (const [key, nested] of Object.entries(value as Record<string, unknown>).slice(0, 16)) {
|
|
out[key] = compactJsonForIssue(nested, depth + 1);
|
|
}
|
|
return out;
|
|
}
|
|
return String(value).slice(0, 240);
|
|
}
|
|
|
|
function compactStepForIssue(value: unknown): Record<string, unknown> | null {
|
|
const step = nullableRecord(value);
|
|
if (step === null) return null;
|
|
return {
|
|
index: typeof step.index === "number" ? step.index : null,
|
|
name: typeof step.name === "string" ? step.name : null,
|
|
ok: typeof step.ok === "boolean" ? step.ok : null,
|
|
atMs: typeof step.atMs === "number" ? step.atMs : null,
|
|
data: compactStepDataForIssue(step.data),
|
|
};
|
|
}
|
|
|
|
function compactStepDataForIssue(value: unknown): unknown {
|
|
const data = nullableRecord(value);
|
|
if (data === null) return compactJsonForIssue(value);
|
|
if (Array.isArray(data.items) && typeof data.failedCount === "number") return compactApiMatrixSummary(data);
|
|
return compactJsonForIssue(data);
|
|
}
|
|
|
|
function compactApiMatrixSummary(value: unknown): Record<string, unknown> | null {
|
|
const matrix = nullableRecord(value);
|
|
if (matrix === null) return null;
|
|
return {
|
|
ok: matrix.ok === true,
|
|
count: typeof matrix.count === "number" ? matrix.count : null,
|
|
okCount: typeof matrix.okCount === "number" ? matrix.okCount : null,
|
|
failedCount: typeof matrix.failedCount === "number" ? matrix.failedCount : null,
|
|
items: Array.isArray(matrix.items)
|
|
? matrix.items.slice(0, 12).map((item) => {
|
|
const row = record(item);
|
|
return {
|
|
name: typeof row.name === "string" ? row.name : null,
|
|
path: typeof row.path === "string" ? row.path : null,
|
|
method: typeof row.method === "string" ? row.method : null,
|
|
ok: row.ok === true,
|
|
status: typeof row.status === "number" ? row.status : null,
|
|
error: typeof row.error === "string" ? row.error : null,
|
|
failureKind: typeof row.failureKind === "string" ? row.failureKind : null,
|
|
bodyKeys: Array.isArray(row.bodyKeys) ? row.bodyKeys.filter((key) => typeof key === "string").slice(0, 12) : [],
|
|
};
|
|
})
|
|
: [],
|
|
};
|
|
}
|
|
|
|
export function compactWebProbeResult(report: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (report === null) return null;
|
|
const dom = record(report.dom);
|
|
const performance = compactWebProbePerformance(report.performance);
|
|
const trace = compactWebProbeTrace(report.trace);
|
|
const session = compactWebProbeSession(report.session);
|
|
const artifacts = record(report.artifacts);
|
|
const promptValidation = compactPromptValidation(report.promptValidation);
|
|
const degradedReason = typeof report.degradedReason === "string" ? report.degradedReason : null;
|
|
const summary = webProbeRunIssueSummary({ report, session, trace, promptValidation, artifacts, degradedReason });
|
|
return {
|
|
ok: report.ok === true,
|
|
status: typeof report.status === "string" ? report.status : null,
|
|
summary,
|
|
finalUrl: typeof report.finalUrl === "string" ? report.finalUrl : null,
|
|
error: typeof report.error === "string" ? report.error : null,
|
|
degradedReason,
|
|
failureKind: typeof summary.failureKind === "string" ? summary.failureKind : null,
|
|
nextAction: typeof summary.nextAction === "string" ? summary.nextAction : null,
|
|
actions: compactWebProbeActions(report.actions),
|
|
session,
|
|
trace,
|
|
promptValidation,
|
|
performance,
|
|
traceSamples: compactWebProbeTraceSamples(report.traceSamples),
|
|
dom: {
|
|
authState: typeof dom.authState === "string" ? dom.authState : null,
|
|
requiredSelectors: record(dom.requiredSelectors),
|
|
messageCount: typeof dom.messageCount === "number" ? dom.messageCount : null,
|
|
sessionRail: record(dom.sessionRail),
|
|
},
|
|
failureDom: record(report.failureDom),
|
|
artifacts,
|
|
safety: record(report.safety),
|
|
};
|
|
}
|
|
|
|
function compactWebProbeSession(value: unknown): Record<string, unknown> {
|
|
const session = record(value);
|
|
const freshSession = record(session.freshSession);
|
|
return {
|
|
freshSessionRequested: session.freshSessionRequested === true,
|
|
freshSessionAligned: typeof session.freshSessionAligned === "boolean" ? session.freshSessionAligned : null,
|
|
conversationId: session.conversationId ?? null,
|
|
routeConversationId: session.routeConversationId ?? null,
|
|
selectedConversationId: session.selectedConversationId ?? null,
|
|
sessionId: session.sessionId ?? null,
|
|
revision: session.revision ?? null,
|
|
messageCount: session.messageCount ?? null,
|
|
degradedReason: session.degradedReason ?? null,
|
|
freshSession: Object.keys(freshSession).length === 0 ? null : {
|
|
settled: freshSession.settled === true,
|
|
aligned: freshSession.aligned === true,
|
|
reason: freshSession.reason ?? null,
|
|
},
|
|
};
|
|
}
|
|
|
|
function compactWebProbeTrace(value: unknown): Record<string, unknown> {
|
|
const trace = record(value);
|
|
const latestFetch = record(trace.latestFetch);
|
|
return {
|
|
requested: trace.requested === true,
|
|
sampleCount: trace.sampleCount ?? null,
|
|
conversationId: trace.conversationId ?? null,
|
|
sessionId: trace.sessionId ?? null,
|
|
traceId: trace.traceId ?? null,
|
|
finalAgentStatus: trace.finalAgentStatus ?? null,
|
|
finalTraceStatus: trace.finalTraceStatus ?? null,
|
|
finalDomRowCount: trace.finalDomRowCount ?? null,
|
|
firstDomRowVisibleAt: trace.firstDomRowVisibleAt ?? null,
|
|
firstDomRowPreview: typeof trace.firstDomRowPreview === "string" ? trace.firstDomRowPreview.slice(0, 240) : null,
|
|
firstRestEventVisibleAt: trace.firstRestEventVisibleAt ?? null,
|
|
firstRestEventCount: trace.firstRestEventCount ?? null,
|
|
restTraceOk: trace.restTraceOk === true,
|
|
failedFetchCount: trace.failedFetchCount ?? null,
|
|
latestFetch: Object.keys(latestFetch).length === 0 ? null : {
|
|
ok: latestFetch.ok === true,
|
|
httpStatus: latestFetch.httpStatus ?? null,
|
|
eventCount: latestFetch.eventCount ?? null,
|
|
traceStatus: latestFetch.traceStatus ?? null,
|
|
degradedReason: latestFetch.degradedReason ?? null,
|
|
attempts: Array.isArray(latestFetch.attempts) ? latestFetch.attempts.length : null,
|
|
latestEventPreview: typeof latestFetch.latestEventPreview === "string" ? latestFetch.latestEventPreview.slice(0, 240) : null,
|
|
},
|
|
degradedReason: trace.degradedReason ?? null,
|
|
};
|
|
}
|
|
|
|
function compactWebProbePerformance(value: unknown): Record<string, unknown> {
|
|
const performance = record(value);
|
|
return {
|
|
startedAt: performance.startedAt ?? null,
|
|
submittedAt: performance.submittedAt ?? null,
|
|
firstDomTraceRowAt: performance.firstDomTraceRowAt ?? null,
|
|
firstRestTraceEventAt: performance.firstRestTraceEventAt ?? null,
|
|
submitToFirstDomTraceRowMs: performance.submitToFirstDomTraceRowMs ?? null,
|
|
submitToFirstRestTraceEventMs: performance.submitToFirstRestTraceEventMs ?? null,
|
|
totalSampleWindowMs: performance.totalSampleWindowMs ?? null,
|
|
};
|
|
}
|
|
|
|
function compactPromptValidation(value: unknown): Record<string, unknown> {
|
|
const prompt = record(value);
|
|
const finalResponse = record(prompt.finalResponse);
|
|
return {
|
|
ok: prompt.ok === true,
|
|
applicable: prompt.applicable === true,
|
|
reason: typeof prompt.reason === "string" ? prompt.reason : null,
|
|
failures: Array.isArray(prompt.failures) ? prompt.failures.filter((item) => typeof item === "string").slice(0, 20) : [],
|
|
finalResponse: Object.keys(finalResponse).length === 0 ? null : {
|
|
textChars: typeof finalResponse.textChars === "number" ? finalResponse.textChars : null,
|
|
textPreview: typeof finalResponse.textPreview === "string" ? finalResponse.textPreview.slice(0, 500) : null,
|
|
markdown: nullableRecord(finalResponse.markdown),
|
|
},
|
|
};
|
|
}
|
|
|
|
function webProbeRunIssueSummary(input: {
|
|
report: Record<string, unknown>;
|
|
session: Record<string, unknown>;
|
|
trace: Record<string, unknown>;
|
|
promptValidation: Record<string, unknown>;
|
|
artifacts: Record<string, unknown>;
|
|
degradedReason: string | null;
|
|
}): Record<string, unknown> {
|
|
const failureKind = webProbeRunFailureKind(input.degradedReason, input.promptValidation);
|
|
const failedCondition = webProbeRunFailedCondition(input.degradedReason, input.promptValidation);
|
|
const promptSubmitted = record(input.report.safety).promptSubmitted === true;
|
|
const traceRequested = input.trace.requested === true || record(input.report.safety).traceSamplingRequested === true;
|
|
return {
|
|
ok: input.report.status === "pass" && !input.degradedReason,
|
|
status: typeof input.report.status === "string" ? input.report.status : null,
|
|
degradedReason: input.degradedReason,
|
|
failureKind,
|
|
failedCondition,
|
|
nextAction: webProbeRunNextAction(failureKind, input.degradedReason, input.promptValidation, promptSubmitted, traceRequested),
|
|
finalUrl: typeof input.report.finalUrl === "string" ? input.report.finalUrl : null,
|
|
conversationId: input.session.conversationId ?? input.trace.conversationId ?? null,
|
|
sessionId: input.session.sessionId ?? input.trace.sessionId ?? null,
|
|
traceId: input.trace.traceId ?? null,
|
|
traceRequested,
|
|
traceStatus: input.trace.finalTraceStatus ?? null,
|
|
agentStatus: input.trace.finalAgentStatus ?? null,
|
|
messageCount: input.session.messageCount ?? record(input.report.dom).messageCount ?? null,
|
|
promptValidation: input.promptValidation,
|
|
reportPath: typeof input.artifacts.reportPath === "string" ? input.artifacts.reportPath : null,
|
|
reportSha256: typeof input.artifacts.reportSha256 === "string" ? input.artifacts.reportSha256 : null,
|
|
screenshotPath: typeof input.artifacts.screenshotPath === "string" ? input.artifacts.screenshotPath : null,
|
|
screenshotSha256: typeof input.artifacts.screenshotSha256 === "string" ? input.artifacts.screenshotSha256 : null,
|
|
traceRecommendation: promptSubmitted && !traceRequested
|
|
? "rerun with --trace-sample-count 2 --trace-sample-interval-ms 1500, or rely on the new default for prompt probes"
|
|
: null,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function webProbeRunFailureKind(degradedReason: string | null, promptValidation: Record<string, unknown>): string | null {
|
|
if (degradedReason === null) return null;
|
|
if (degradedReason === "prompt-validation-failed") return "unmet-expectation";
|
|
if (degradedReason === "agent-terminal-timeout") return "agent-runtime-timeout";
|
|
if (/trace-fetch|api|fetch|http|network/iu.test(degradedReason)) return "network-or-api-fetch-bug";
|
|
if (/auth|login|credential/iu.test(degradedReason)) return "target-auth-bug";
|
|
if (/browser|timeout|playwright|chromium/iu.test(degradedReason)) return "browser-environment-bug";
|
|
const failures = Array.isArray(promptValidation.failures) ? promptValidation.failures.join(" ") : "";
|
|
if (/final-response|agent-message|markdown|completed/iu.test(failures)) return "unmet-expectation";
|
|
return "user-facing-web-bug";
|
|
}
|
|
|
|
function webProbeRunFailedCondition(degradedReason: string | null, promptValidation: Record<string, unknown>): string | null {
|
|
if (degradedReason === "prompt-validation-failed") {
|
|
const failures = Array.isArray(promptValidation.failures) ? promptValidation.failures.filter((item) => typeof item === "string") : [];
|
|
return failures.length > 0 ? failures.join(",") : "prompt final response validation failed";
|
|
}
|
|
return degradedReason;
|
|
}
|
|
|
|
function webProbeRunNextAction(
|
|
failureKind: string | null,
|
|
degradedReason: string | null,
|
|
promptValidation: Record<string, unknown>,
|
|
promptSubmitted: boolean,
|
|
traceRequested: boolean,
|
|
): string | null {
|
|
if (failureKind === null) return null;
|
|
if (failureKind === "unmet-expectation") {
|
|
const failures = Array.isArray(promptValidation.failures) ? promptValidation.failures.join(",") : "";
|
|
if (/agent-not-completed/iu.test(failures)) return "Increase --wait-agent-terminal-ms or inspect the Code Agent terminal state for the conversation/trace ids in summary.";
|
|
if (/final-response/iu.test(failures)) return "Inspect Workbench message projection and conversation detail; terminal messages must expose a final response.";
|
|
if (/markdown/iu.test(failures)) return "Check whether the prompt expectation still requires markdown structure; stale assertions should be removed instead of preserved.";
|
|
return "Inspect promptValidation.failures, screenshots, and reportPath to decide whether the Web behavior or the assertion is stale.";
|
|
}
|
|
if (failureKind === "network-or-api-fetch-bug") return "Inspect trace/session API fetch fields in reportPath and retry after checking target API availability.";
|
|
if (failureKind === "agent-runtime-timeout") return "Inspect summary.traceId/sessionId and the corresponding Code Agent/AgentRun runtime; the browser probe submitted the prompt and observed trace events, but no terminal agent response arrived before the wait boundary.";
|
|
if (failureKind === "target-auth-bug") return "Inspect credential sourceRef/fingerprint and /auth/login status; do not print secrets.";
|
|
if (failureKind === "browser-environment-bug") return "Inspect browser launcher and Playwright availability on the target workspace.";
|
|
if (promptSubmitted && !traceRequested) return "Rerun with trace sampling if trace evidence is required.";
|
|
return degradedReason === null ? null : "Inspect reportPath for full redacted details, then rerun the same node/lane entry.";
|
|
}
|
|
|
|
function compactWebProbeTraceSamples(value: unknown): Record<string, unknown>[] {
|
|
if (!Array.isArray(value)) return [];
|
|
return value.map(record).map((sample) => {
|
|
const traceFetch = record(sample.traceFetch);
|
|
const attempts = Array.isArray(traceFetch.attempts) ? traceFetch.attempts : [];
|
|
return {
|
|
index: sample.index ?? null,
|
|
sampledAt: sample.sampledAt ?? null,
|
|
messageCount: sample.messageCount ?? null,
|
|
agentStatus: sample.agentStatus ?? null,
|
|
tracePresent: sample.tracePresent === true,
|
|
traceStatus: sample.traceStatus ?? null,
|
|
rowCount: sample.rowCount ?? null,
|
|
emptyLabel: sample.emptyLabel ?? null,
|
|
latestRowPreview: sample.latestRowPreview ?? null,
|
|
userVisibleRowCount: sample.userVisibleRowCount ?? null,
|
|
latestUserVisibleRowPreview: sample.latestUserVisibleRowPreview ?? null,
|
|
conversationId: sample.conversationId ?? null,
|
|
sessionId: sample.sessionId ?? null,
|
|
traceId: sample.traceId ?? null,
|
|
traceFetch: {
|
|
ok: traceFetch.ok === true,
|
|
httpStatus: traceFetch.httpStatus ?? null,
|
|
eventCount: traceFetch.eventCount ?? null,
|
|
degradedReason: traceFetch.degradedReason ?? null,
|
|
attempts: attempts.length,
|
|
latestEventPreview: traceFetch.latestEventPreview ?? null,
|
|
},
|
|
};
|
|
});
|
|
}
|
|
|
|
function compactWebProbeActions(value: unknown): Record<string, unknown>[] {
|
|
if (!Array.isArray(value)) return [];
|
|
return value.map(record).map((action) => {
|
|
const name = typeof action.action === "string" ? action.action : "unknown";
|
|
if (name === "fresh-session") {
|
|
const alignment = record(action.alignment);
|
|
const check = record(alignment.check);
|
|
const repair = record(alignment.repair);
|
|
return {
|
|
action: name,
|
|
settled: action.settled === true,
|
|
aligned: action.aligned === true,
|
|
reason: alignment.reason ?? check.reason ?? null,
|
|
conversationId: check.conversationId ?? null,
|
|
routeConversationId: check.routeConversationId ?? null,
|
|
selectedConversationId: check.selectedConversationId ?? null,
|
|
repaired: Object.keys(repair).length > 0,
|
|
repairConversationId: repair.conversationId ?? null,
|
|
attempts: Array.isArray(alignment.attempts) ? alignment.attempts.length : null,
|
|
};
|
|
}
|
|
if (name === "trace-interval-samples") {
|
|
return {
|
|
action: name,
|
|
count: action.count ?? null,
|
|
intervalMs: action.intervalMs ?? null,
|
|
};
|
|
}
|
|
if (name === "submit-prompt") {
|
|
return {
|
|
action: name,
|
|
chars: action.chars ?? null,
|
|
submittedAt: action.submittedAt ?? null,
|
|
};
|
|
}
|
|
if (name === "wait-agent-terminal") {
|
|
return {
|
|
action: name,
|
|
terminal: action.terminal === true,
|
|
timeoutMs: action.timeoutMs ?? null,
|
|
};
|
|
}
|
|
return {
|
|
action: name,
|
|
skipped: action.skipped ?? null,
|
|
ready: action.ready ?? null,
|
|
found: action.found ?? null,
|
|
timeoutMs: action.timeoutMs ?? null,
|
|
};
|
|
});
|
|
}
|