fix: recover blank control projection during prompts
This commit is contained in:
@@ -654,6 +654,9 @@ lanes:
|
|||||||
prometheusOperator: false
|
prometheusOperator: false
|
||||||
webProbe:
|
webProbe:
|
||||||
sentinels:
|
sentinels:
|
||||||
|
- id: workbench-dsflash-go-tool-call-10x
|
||||||
|
enabled: true
|
||||||
|
configRef: config/hwlab-web-probe-sentinels/d518-v03/workbench-dsflash-go-tool-call-10x.yaml#sentinel
|
||||||
- id: workbench-fake-echo-session-invariance-10x
|
- id: workbench-fake-echo-session-invariance-10x
|
||||||
enabled: true
|
enabled: true
|
||||||
configRef: config/hwlab-web-probe-sentinels/d518-v03/workbench-fake-echo-session-invariance-10x.yaml#sentinel
|
configRef: config/hwlab-web-probe-sentinels/d518-v03/workbench-fake-echo-session-invariance-10x.yaml#sentinel
|
||||||
|
|||||||
@@ -221,6 +221,19 @@ function userMessageFor(items,prompt){
|
|||||||
}
|
}
|
||||||
return {preview:short(prompt?.input?.textPreview||'',90)||'(hash only)',textHash:hash,textBytes:prompt?.input?.textBytes??null};
|
return {preview:short(prompt?.input?.textPreview||'',90)||'(hash only)',textHash:hash,textBytes:prompt?.input?.textBytes??null};
|
||||||
}
|
}
|
||||||
|
function promptFailedBeforeTrace(prompt,segment){
|
||||||
|
if(prompt?.phase!=='failed')return false;
|
||||||
|
if(commandTraceId(prompt)||prompt?.traceId)return false;
|
||||||
|
const hash=prompt?.input?.textHash||null;
|
||||||
|
if(hash){
|
||||||
|
for(const sample of segment){
|
||||||
|
for(const message of Array.isArray(sample.messages)?sample.messages:[]){
|
||||||
|
if(message?.textHash===hash)return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
function cleanFinalResponseText(text){
|
function cleanFinalResponseText(text){
|
||||||
const raw=String(text||'').trim();
|
const raw=String(text||'').trim();
|
||||||
if(!raw) return '';
|
if(!raw) return '';
|
||||||
@@ -284,13 +297,13 @@ function turnSummaryRows(){
|
|||||||
return [{round:0,commandId:null,commandType:null,userPreview:'(无 sendPrompt control 记录)',userHash:null,userBytes:null,traceId:allTraceIds[0]||null,status:statusFor(samples,allTraceIds[0]||null),elapsedSeconds:null,recentUpdateSeconds:null,marks:'-',firstSeq:samples[0]?.seq??null,lastSeq:samples[samples.length-1]?.seq??null,lastTs:samples[samples.length-1]?.ts??null,finalResponse:finalResponseFor(samples,allTraceIds[0]||null),valuesRedacted:true}];
|
return [{round:0,commandId:null,commandType:null,userPreview:'(无 sendPrompt control 记录)',userHash:null,userBytes:null,traceId:allTraceIds[0]||null,status:statusFor(samples,allTraceIds[0]||null),elapsedSeconds:null,recentUpdateSeconds:null,marks:'-',firstSeq:samples[0]?.seq??null,lastSeq:samples[samples.length-1]?.seq??null,lastTs:samples[samples.length-1]?.ts??null,finalResponse:finalResponseFor(samples,allTraceIds[0]||null),valuesRedacted:true}];
|
||||||
}
|
}
|
||||||
return prompts.map((prompt,index)=>{
|
return prompts.map((prompt,index)=>{
|
||||||
const segment=segmentFor(index,prompts); const segmentControls=controlsFor(index,prompts); const selected=choosePrimaryTraceId(index,segment,prompt); const traceId=selected.traceId; const user=userMessageFor(segment,prompt);
|
const segment=segmentFor(index,prompts); const segmentControls=controlsFor(index,prompts); const failedBeforeTrace=promptFailedBeforeTrace(prompt,segment); const selected=failedBeforeTrace?{traceId:null,columns:[]}:choosePrimaryTraceId(index,segment,prompt); const traceId=selected.traceId; const user=userMessageFor(segment,prompt);
|
||||||
const primaryColumn=selected.columns.find((column)=>column.traceId===traceId)||null;
|
const primaryColumn=selected.columns.find((column)=>column.traceId===traceId)||null;
|
||||||
const traceSample=latestSampleForTrace(segment,traceId,primaryColumn?.lastSeq??null);
|
const traceSample=latestSampleForTrace(segment,traceId,primaryColumn?.lastSeq??null);
|
||||||
const texts=textsFor(segment,traceId);
|
const texts=textsFor(segment,traceId);
|
||||||
const elapsedValues=texts.map(parseElapsed).filter((value)=>value!==null); const recentValues=texts.map(parseRecent).filter((value)=>value!==null);
|
const elapsedValues=texts.map(parseElapsed).filter((value)=>value!==null); const recentValues=texts.map(parseRecent).filter((value)=>value!==null);
|
||||||
const marks=unique(segmentControls.map((item)=>item.type==='cancel'?'cancel':item.type==='steer'?'steer':null)).join(',')||'-';
|
const marks=unique([...segmentControls.map((item)=>item.type==='cancel'?'cancel':item.type==='steer'?'steer':null),failedBeforeTrace?'command-failed':null]).join(',')||'-';
|
||||||
return {round:index+1,commandId:prompt.commandId||null,commandType:prompt.type||null,sessionId:prompt.sessionId||null,userPreview:user.preview,userHash:user.textHash,userBytes:user.textBytes,traceId,traceIds:unique([traceId,...selected.columns.map((column)=>column.traceId),...traceIdsFromSamples(segment)]).slice(0,8),status:statusFor(segment,traceId),elapsedSeconds:elapsedValues.length?Math.max(...elapsedValues):null,recentUpdateSeconds:recentValues.length?Math.max(...recentValues):null,marks,firstSeq:segment[0]?.seq??primaryColumn?.firstSeq??null,lastSeq:traceSample?.seq??primaryColumn?.lastSeq??segment[segment.length-1]?.seq??null,lastTs:traceSample?.ts??segment[segment.length-1]?.ts??null,finalResponse:finalResponseFor(segment,traceId,user),valuesRedacted:true};
|
return {round:index+1,commandId:prompt.commandId||null,commandType:prompt.type||null,sessionId:prompt.sessionId||null,userPreview:user.preview,userHash:user.textHash,userBytes:user.textBytes,traceId,traceIds:failedBeforeTrace?[]:unique([traceId,...selected.columns.map((column)=>column.traceId),...traceIdsFromSamples(segment)]).slice(0,8),status:failedBeforeTrace?'command-failed':statusFor(segment,traceId),elapsedSeconds:failedBeforeTrace?null:(elapsedValues.length?Math.max(...elapsedValues):null),recentUpdateSeconds:failedBeforeTrace?null:(recentValues.length?Math.max(...recentValues):null),marks,firstSeq:segment[0]?.seq??primaryColumn?.firstSeq??null,lastSeq:failedBeforeTrace?(segment[segment.length-1]?.seq??null):(traceSample?.seq??primaryColumn?.lastSeq??segment[segment.length-1]?.seq??null),lastTs:failedBeforeTrace?(segment[segment.length-1]?.ts??null):(traceSample?.ts??segment[segment.length-1]?.ts??null),finalResponse:failedBeforeTrace?emptyFinalResponse():finalResponseFor(segment,traceId,user),valuesRedacted:true};
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
function pad(value,width){const text=short(value,width); return text+Array(Math.max(0,width-text.length)+1).join(' ')}
|
function pad(value,width){const text=short(value,width); return text+Array(Math.max(0,width-text.length)+1).join(' ')}
|
||||||
|
|||||||
@@ -442,6 +442,11 @@ async function withObserverSync(result, reason) {
|
|||||||
async function syncObserverPageToControlSession(reason, explicitSessionId = null, options = {}) {
|
async function syncObserverPageToControlSession(reason, explicitSessionId = null, options = {}) {
|
||||||
if (!observerPage || observerPage.isClosed()) return { ok: false, reason, pageRole: "observer", pageId: observerPageId, failureKind: "observer-page-unavailable" };
|
if (!observerPage || observerPage.isClosed()) return { ok: false, reason, pageRole: "observer", pageId: observerPageId, failureKind: "observer-page-unavailable" };
|
||||||
const forceRefresh = options?.forceRefresh === true;
|
const forceRefresh = options?.forceRefresh === true;
|
||||||
|
const navigationTimeoutMs = Number.isFinite(Number(options.navigationTimeoutMs)) ? Math.max(1, Number(options.navigationTimeoutMs)) : 45000;
|
||||||
|
const readinessTimeoutMs = Number.isFinite(Number(options.readinessTimeoutMs)) ? Math.max(1, Number(options.readinessTimeoutMs)) : 15000;
|
||||||
|
const hydrationTimeoutMs = Number.isFinite(Number(options.hydrationTimeoutMs)) ? Math.max(1, Number(options.hydrationTimeoutMs)) : 15000;
|
||||||
|
const shortCircuitReadinessTimeoutMs = Number.isFinite(Number(options.shortCircuitReadinessTimeoutMs)) ? Math.max(1, Number(options.shortCircuitReadinessTimeoutMs)) : 1000;
|
||||||
|
const shortCircuitHydrationTimeoutMs = Number.isFinite(Number(options.shortCircuitHydrationTimeoutMs)) ? Math.max(1, Number(options.shortCircuitHydrationTimeoutMs)) : 1000;
|
||||||
const snapshot = await workbenchSessionSnapshot();
|
const snapshot = await workbenchSessionSnapshot();
|
||||||
const sessionId = explicitSessionId || snapshot?.activeSessionId || snapshot?.routeSessionId || routeSessionIdFromUrl(currentPageUrl());
|
const sessionId = explicitSessionId || snapshot?.activeSessionId || snapshot?.routeSessionId || routeSessionIdFromUrl(currentPageUrl());
|
||||||
const target = sessionId ? "/workbench/sessions/" + encodeURIComponent(sessionId) : targetPath;
|
const target = sessionId ? "/workbench/sessions/" + encodeURIComponent(sessionId) : targetPath;
|
||||||
@@ -450,7 +455,7 @@ async function syncObserverPageToControlSession(reason, explicitSessionId = null
|
|||||||
const beforeSessionId = routeSessionIdFromUrl(beforeUrl);
|
const beforeSessionId = routeSessionIdFromUrl(beforeUrl);
|
||||||
const attempts = [];
|
const attempts = [];
|
||||||
if (sessionId && beforeSessionId === sessionId && !forceRefresh) {
|
if (sessionId && beforeSessionId === sessionId && !forceRefresh) {
|
||||||
const current = await observerSessionReadiness(targetUrl, sessionId, { readinessTimeoutMs: 1000, hydrationTimeoutMs: 1000 });
|
const current = await observerSessionReadiness(targetUrl, sessionId, { readinessTimeoutMs: shortCircuitReadinessTimeoutMs, hydrationTimeoutMs: shortCircuitHydrationTimeoutMs });
|
||||||
if (current.ok) return { ok: true, reason, changed: false, observerRoundTrip: false, sessionId, beforeUrl, afterUrl: beforeUrl, pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, readiness: current.readiness, hydration: current.hydration, valuesRedacted: true };
|
if (current.ok) return { ok: true, reason, changed: false, observerRoundTrip: false, sessionId, beforeUrl, afterUrl: beforeUrl, pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, readiness: current.readiness, hydration: current.hydration, valuesRedacted: true };
|
||||||
attempts.push({ attempt: 0, ok: false, shortCircuitRejected: true, failureKind: current.failureKind, readiness: current.readiness, hydration: current.hydration, beforeUrl, afterUrl: pageUrl(observerPage), valuesRedacted: true });
|
attempts.push({ attempt: 0, ok: false, shortCircuitRejected: true, failureKind: current.failureKind, readiness: current.readiness, hydration: current.hydration, beforeUrl, afterUrl: pageUrl(observerPage), valuesRedacted: true });
|
||||||
}
|
}
|
||||||
@@ -460,7 +465,7 @@ async function syncObserverPageToControlSession(reason, explicitSessionId = null
|
|||||||
observerPageEpoch += 1;
|
observerPageEpoch += 1;
|
||||||
let status = null;
|
let status = null;
|
||||||
let statusText = null;
|
let statusText = null;
|
||||||
const response = await observerPage.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 45000 }).catch((error) => ({ observerGotoError: errorSummary(error) }));
|
const response = await observerPage.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: navigationTimeoutMs }).catch((error) => ({ observerGotoError: errorSummary(error) }));
|
||||||
if (response?.observerGotoError) {
|
if (response?.observerGotoError) {
|
||||||
attempts.push({ attempt, ok: false, failureKind: navigationFailureKind(response.observerGotoError?.message || response.observerGotoError?.name || "observer-navigation-error"), beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), error: response.observerGotoError, valuesRedacted: true });
|
attempts.push({ attempt, ok: false, failureKind: navigationFailureKind(response.observerGotoError?.message || response.observerGotoError?.name || "observer-navigation-error"), beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), error: response.observerGotoError, valuesRedacted: true });
|
||||||
if (attempt < maxAttempts && isRetryableNavigationError(response.observerGotoError?.message || response.observerGotoError?.name || "")) {
|
if (attempt < maxAttempts && isRetryableNavigationError(response.observerGotoError?.message || response.observerGotoError?.name || "")) {
|
||||||
@@ -472,7 +477,7 @@ async function syncObserverPageToControlSession(reason, explicitSessionId = null
|
|||||||
}
|
}
|
||||||
status = typeof response?.status === "function" ? response.status() : null;
|
status = typeof response?.status === "function" ? response.status() : null;
|
||||||
statusText = typeof response?.statusText === "function" ? response.statusText() : null;
|
statusText = typeof response?.statusText === "function" ? response.statusText() : null;
|
||||||
const readiness = await waitForTargetPageReady(observerPage, targetUrl, { timeoutMs: 15000 });
|
const readiness = await waitForTargetPageReady(observerPage, targetUrl, { timeoutMs: readinessTimeoutMs });
|
||||||
if (!readiness.ok) {
|
if (!readiness.ok) {
|
||||||
attempts.push({ attempt, ok: false, failureKind: readiness.reason || "observer-target-not-ready", beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), httpStatus: status, statusText, readiness, valuesRedacted: true });
|
attempts.push({ attempt, ok: false, failureKind: readiness.reason || "observer-target-not-ready", beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), httpStatus: status, statusText, readiness, valuesRedacted: true });
|
||||||
if (attempt < maxAttempts && observerReadinessRetryable(readiness)) {
|
if (attempt < maxAttempts && observerReadinessRetryable(readiness)) {
|
||||||
@@ -487,7 +492,7 @@ async function syncObserverPageToControlSession(reason, explicitSessionId = null
|
|||||||
attempts.push({ attempt, ok: true, beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), httpStatus: status, statusText, readiness, valuesRedacted: true });
|
attempts.push({ attempt, ok: true, beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), httpStatus: status, statusText, readiness, valuesRedacted: true });
|
||||||
return { ok: true, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, httpStatus: status, statusText, readiness, hydration: null, attempts, valuesRedacted: true };
|
return { ok: true, reason, changed: true, observerRoundTrip: forceRefresh, sessionId: null, targetPath: target, beforeUrl, afterUrl: pageUrl(observerPage), pageRole: "observer", pageId: observerPageId, pageEpoch: observerPageEpoch, httpStatus: status, statusText, readiness, hydration: null, attempts, valuesRedacted: true };
|
||||||
}
|
}
|
||||||
const hydration = await waitForWorkbenchSessionHydrated(observerPage, sessionId, { timeoutMs: 15000 });
|
const hydration = await waitForWorkbenchSessionHydrated(observerPage, sessionId, { timeoutMs: hydrationTimeoutMs });
|
||||||
attempts.push({ attempt, ok: hydration.ok === true, failureKind: hydration.ok === true ? null : hydration.reason || "observer-session-hydration-failed", beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), httpStatus: status, statusText, readiness, hydration, valuesRedacted: true });
|
attempts.push({ attempt, ok: hydration.ok === true, failureKind: hydration.ok === true ? null : hydration.reason || "observer-session-hydration-failed", beforeUrl: attemptBeforeUrl, afterUrl: pageUrl(observerPage), httpStatus: status, statusText, readiness, hydration, valuesRedacted: true });
|
||||||
if (hydration.ok === true) {
|
if (hydration.ok === true) {
|
||||||
lastObserverRefreshAtMs = Date.now();
|
lastObserverRefreshAtMs = Date.now();
|
||||||
@@ -1056,16 +1061,21 @@ function publicProxyServer(raw) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function gotoTarget(rawTarget) {
|
async function gotoTarget(rawTarget, options = {}) {
|
||||||
const target = new URL(String(rawTarget || targetPath), baseUrl).toString();
|
const target = new URL(String(rawTarget || targetPath), baseUrl).toString();
|
||||||
const beforeUrl = currentPageUrl();
|
const beforeUrl = currentPageUrl();
|
||||||
const attempts = [];
|
const attempts = [];
|
||||||
for (let attempt = 1; attempt <= navigationMaxAttempts; attempt += 1) {
|
const maxAttempts = Number.isFinite(Number(options.maxAttempts)) ? Math.max(1, Number(options.maxAttempts)) : navigationMaxAttempts;
|
||||||
|
const navigationTimeoutMs = Number.isFinite(Number(options.navigationTimeoutMs)) ? Math.max(1, Number(options.navigationTimeoutMs)) : 45000;
|
||||||
|
const readinessTimeoutMs = Number.isFinite(Number(options.readinessTimeoutMs)) ? Math.max(1, Number(options.readinessTimeoutMs)) : 15000;
|
||||||
|
const settleMs = Number.isFinite(Number(options.settleMs)) ? Math.max(0, Number(options.settleMs)) : 1000;
|
||||||
|
const lateReadinessTimeoutMs = Number.isFinite(Number(options.lateReadinessTimeoutMs)) ? Math.max(0, Number(options.lateReadinessTimeoutMs)) : 5000;
|
||||||
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||||
try {
|
try {
|
||||||
const response = await page.goto(target, { waitUntil: "domcontentloaded", timeout: 45000 });
|
const response = await page.goto(target, { waitUntil: "domcontentloaded", timeout: navigationTimeoutMs });
|
||||||
await page.waitForTimeout(1000).catch(() => {});
|
if (settleMs > 0) await page.waitForTimeout(settleMs).catch(() => {});
|
||||||
const httpStatus = response ? response.status() : null;
|
const httpStatus = response ? response.status() : null;
|
||||||
const readiness = await waitForTargetPageReady(page, target, { timeoutMs: 15000 });
|
const readiness = await waitForTargetPageReady(page, target, { timeoutMs: readinessTimeoutMs });
|
||||||
if (!readiness.ok) {
|
if (!readiness.ok) {
|
||||||
const pageProvenance = await refreshPageProvenance("goto-degraded", httpStatus).catch(() => null);
|
const pageProvenance = await refreshPageProvenance("goto-degraded", httpStatus).catch(() => null);
|
||||||
attempts.push({ attempt, ok: false, degraded: true, httpStatus, readiness, failureKind: readiness.reason || "workbench-app-not-ready" });
|
attempts.push({ attempt, ok: false, degraded: true, httpStatus, readiness, failureKind: readiness.reason || "workbench-app-not-ready" });
|
||||||
@@ -1077,15 +1087,15 @@ async function gotoTarget(rawTarget) {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message = error instanceof Error ? error.message : String(error);
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
attempts.push({ attempt, ok: false, failureKind: navigationFailureKind(message), message: redactErrorMessage(message), readiness: error?.navigationReadiness ?? null });
|
attempts.push({ attempt, ok: false, failureKind: navigationFailureKind(message), message: redactErrorMessage(message), readiness: error?.navigationReadiness ?? null });
|
||||||
if (/workbench-app-not-ready|navigation timeout|page\.goto:\s*timeout|timeout\s+\d+ms\s+exceeded/iu.test(message)) {
|
if (lateReadinessTimeoutMs > 0 && /workbench-app-not-ready|navigation timeout|page\.goto:\s*timeout|timeout\s+\d+ms\s+exceeded/iu.test(message)) {
|
||||||
const lateReadiness = await waitForTargetPageReady(page, target, { timeoutMs: 5000 }).catch(() => null);
|
const lateReadiness = await waitForTargetPageReady(page, target, { timeoutMs: lateReadinessTimeoutMs }).catch(() => null);
|
||||||
if (lateReadiness?.ok) {
|
if (lateReadiness?.ok) {
|
||||||
const pageProvenance = await refreshPageProvenance("goto-late-ready", null);
|
const pageProvenance = await refreshPageProvenance("goto-late-ready", null);
|
||||||
attempts.push({ attempt, ok: true, lateReady: true, httpStatus: null, readiness: lateReadiness });
|
attempts.push({ attempt, ok: true, lateReady: true, httpStatus: null, readiness: lateReadiness });
|
||||||
return { beforeUrl, afterUrl: currentPageUrl(), httpStatus: null, pageId, pageProvenance: compactPageProvenance(pageProvenance), readiness: lateReadiness, attempts };
|
return { beforeUrl, afterUrl: currentPageUrl(), httpStatus: null, pageId, pageProvenance: compactPageProvenance(pageProvenance), readiness: lateReadiness, attempts };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (attempt >= navigationMaxAttempts || !isRetryableNavigationError(message)) {
|
if (attempt >= maxAttempts || !isRetryableNavigationError(message)) {
|
||||||
throw Object.assign(new Error(message), { attempts, target });
|
throw Object.assign(new Error(message), { attempts, target });
|
||||||
}
|
}
|
||||||
if (!observerPage) {
|
if (!observerPage) {
|
||||||
@@ -1628,6 +1638,18 @@ function controlPageRecoveryTarget(snapshot, beforeUrl) {
|
|||||||
return { sessionId: null, targetPath, valuesRedacted: true };
|
return { sessionId: null, targetPath, valuesRedacted: true };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function controlPageProjectionMissingForCommand(snapshot, beforeUrl) {
|
||||||
|
const path = safeUrlPath(snapshot?.url || beforeUrl);
|
||||||
|
if (!isWorkbenchPathname(path || "")) return false;
|
||||||
|
const routeSessionId = snapshot?.routeSessionId || routeSessionIdFromUrl(snapshot?.url || beforeUrl);
|
||||||
|
if (!routeSessionId) return false;
|
||||||
|
return snapshot?.activeSessionId !== routeSessionId
|
||||||
|
&& Number(snapshot?.tabCount || 0) === 0
|
||||||
|
&& Number(snapshot?.messageCount || 0) === 0
|
||||||
|
&& Number(snapshot?.traceRowCount || 0) === 0
|
||||||
|
&& snapshot?.composerReady !== true;
|
||||||
|
}
|
||||||
|
|
||||||
async function controlPageLivenessSnapshot(reason, timeoutMs = 1500) {
|
async function controlPageLivenessSnapshot(reason, timeoutMs = 1500) {
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
return withHardTimeout(workbenchSessionSnapshot(page), timeoutMs, "control page liveness snapshot exceeded " + timeoutMs + "ms")
|
return withHardTimeout(workbenchSessionSnapshot(page), timeoutMs, "control page liveness snapshot exceeded " + timeoutMs + "ms")
|
||||||
@@ -1653,24 +1675,28 @@ async function controlPageLivenessSnapshot(reason, timeoutMs = 1500) {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function recoverControlPageToTarget(reason, beforeUrl, target, liveness = null) {
|
async function recoverControlPageToTarget(reason, beforeUrl, target, liveness = null, options = {}) {
|
||||||
let navigation = null;
|
let navigation = null;
|
||||||
let hydration = null;
|
let hydration = null;
|
||||||
let afterLiveness = null;
|
let afterLiveness = null;
|
||||||
const attempts = [];
|
const attempts = [];
|
||||||
let ok = false;
|
let ok = false;
|
||||||
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
const maxAttempts = Number.isFinite(Number(options.maxAttempts)) ? Math.max(1, Number(options.maxAttempts)) : 2;
|
||||||
|
const hydrationTimeoutMs = Number.isFinite(Number(options.hydrationTimeoutMs)) ? Math.max(1, Number(options.hydrationTimeoutMs)) : 12000;
|
||||||
|
const hydrationHardTimeoutMs = Number.isFinite(Number(options.hydrationHardTimeoutMs)) ? Math.max(hydrationTimeoutMs, Number(options.hydrationHardTimeoutMs)) : hydrationTimeoutMs + 2000;
|
||||||
|
const navigationOptions = options.navigation || {};
|
||||||
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||||
await recreateControlPageForNavigation(reason + "-control-page-recovery", attempt);
|
await recreateControlPageForNavigation(reason + "-control-page-recovery", attempt);
|
||||||
try {
|
try {
|
||||||
navigation = await gotoTarget(target.targetPath);
|
navigation = await gotoTarget(target.targetPath, navigationOptions);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
navigation = { ok: false, targetPath: target.targetPath, error: errorSummary(error), valuesRedacted: true };
|
navigation = { ok: false, targetPath: target.targetPath, error: errorSummary(error), valuesRedacted: true };
|
||||||
}
|
}
|
||||||
if (!navigation?.error && target.sessionId) {
|
if (!navigation?.error && target.sessionId) {
|
||||||
hydration = await withHardTimeout(
|
hydration = await withHardTimeout(
|
||||||
waitForWorkbenchSessionHydrated(page, target.sessionId, { timeoutMs: 12000 }),
|
waitForWorkbenchSessionHydrated(page, target.sessionId, { timeoutMs: hydrationTimeoutMs }),
|
||||||
14000,
|
hydrationHardTimeoutMs,
|
||||||
"control page recovery hydration exceeded 14000ms"
|
"control page recovery hydration exceeded " + hydrationHardTimeoutMs + "ms"
|
||||||
).catch((error) => ({ ok: false, error: errorSummary(error), valuesRedacted: true }));
|
).catch((error) => ({ ok: false, error: errorSummary(error), valuesRedacted: true }));
|
||||||
} else {
|
} else {
|
||||||
hydration = null;
|
hydration = null;
|
||||||
@@ -1699,12 +1725,73 @@ async function recoverControlPageToTarget(reason, beforeUrl, target, liveness =
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function promoteObserverPageToControlForCommand(reason, target, liveness = null) {
|
||||||
|
const beforeUrl = currentPageUrl();
|
||||||
|
const beforeObserverUrl = pageUrl(observerPage);
|
||||||
|
if (!observerPage || observerPage.isClosed()) {
|
||||||
|
return { ok: false, promoted: false, reason, failureKind: "observer-page-unavailable", beforeUrl, observerUrl: beforeObserverUrl, target, liveness, valuesRedacted: true };
|
||||||
|
}
|
||||||
|
const sessionId = target?.sessionId || routeSessionIdFromUrl(beforeUrl);
|
||||||
|
if (!sessionId) {
|
||||||
|
return { ok: false, promoted: false, reason, failureKind: "observer-promotion-needs-session", beforeUrl, observerUrl: beforeObserverUrl, target, liveness, valuesRedacted: true };
|
||||||
|
}
|
||||||
|
const observerSessionId = routeSessionIdFromUrl(beforeObserverUrl);
|
||||||
|
if (observerSessionId !== sessionId) {
|
||||||
|
return { ok: false, promoted: false, reason, failureKind: "observer-session-mismatch", beforeUrl, observerUrl: beforeObserverUrl, observerSessionId, sessionId, target, liveness, valuesRedacted: true };
|
||||||
|
}
|
||||||
|
const targetUrl = new URL(target?.targetPath || ("/workbench/sessions/" + encodeURIComponent(sessionId)), baseUrl).toString();
|
||||||
|
const readiness = await observerSessionReadiness(targetUrl, sessionId, { readinessTimeoutMs: 1000, hydrationTimeoutMs: 2000 });
|
||||||
|
const observerComposerReady = readiness?.hydration?.snapshot?.composerReady === true;
|
||||||
|
if (readiness.ok !== true || observerComposerReady !== true) {
|
||||||
|
return { ok: false, promoted: false, reason, failureKind: readiness.failureKind || (observerComposerReady ? "observer-not-ready" : "observer-composer-not-ready"), beforeUrl, observerUrl: beforeObserverUrl, sessionId, target, liveness, readiness, valuesRedacted: true };
|
||||||
|
}
|
||||||
|
const oldControlPage = page;
|
||||||
|
observerPageEpoch += 1;
|
||||||
|
controlPageEpoch += 1;
|
||||||
|
page = observerPage;
|
||||||
|
observerPage = null;
|
||||||
|
attachPassiveListeners(page, "control", pageId);
|
||||||
|
currentPageProvenance = null;
|
||||||
|
if (oldControlPage && !oldControlPage.isClosed() && oldControlPage !== page) {
|
||||||
|
await withHardTimeout(oldControlPage.close(), 2000, "old control page close exceeded 2000ms")
|
||||||
|
.catch((error) => appendJsonl(files.errors, eventRecord("old-control-page-close-timeout", { reason, error: errorSummary(error), pageRole: "control", pageId, pageEpoch: controlPageEpoch })));
|
||||||
|
}
|
||||||
|
observerPage = await context.newPage();
|
||||||
|
attachPassiveListeners(observerPage, "observer", observerPageId);
|
||||||
|
const observerSync = await syncObserverPageToControlSession(reason + "-observer-recreated-after-promotion", sessionId, {
|
||||||
|
maxAttempts: 1,
|
||||||
|
navigationTimeoutMs: 8000,
|
||||||
|
readinessTimeoutMs: 3000,
|
||||||
|
hydrationTimeoutMs: 3000,
|
||||||
|
shortCircuitReadinessTimeoutMs: 1000,
|
||||||
|
shortCircuitHydrationTimeoutMs: 1000,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
promoted: true,
|
||||||
|
reason,
|
||||||
|
beforeUrl,
|
||||||
|
beforeObserverUrl,
|
||||||
|
afterUrl: currentPageUrl(),
|
||||||
|
sessionId,
|
||||||
|
target,
|
||||||
|
liveness,
|
||||||
|
readiness,
|
||||||
|
observerSync,
|
||||||
|
pageRole: "control",
|
||||||
|
pageId,
|
||||||
|
pageEpoch: controlPageEpoch,
|
||||||
|
valuesRedacted: true
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function ensureControlPageResponsiveForCommand(reason) {
|
async function ensureControlPageResponsiveForCommand(reason) {
|
||||||
const beforeUrl = currentPageUrl();
|
const beforeUrl = currentPageUrl();
|
||||||
const liveness = await controlPageLivenessSnapshot(reason + "-preflight", 3000);
|
const liveness = await controlPageLivenessSnapshot(reason + "-preflight", 3000);
|
||||||
if (liveness.ok) return { ok: true, recovered: false, reason, beforeUrl, afterUrl: currentPageUrl(), liveness, pageRole: "control", pageId, pageEpoch: controlPageEpoch, valuesRedacted: true };
|
const projectionMissing = liveness.ok === true && controlPageProjectionMissingForCommand(liveness.snapshot, beforeUrl);
|
||||||
|
if (liveness.ok && !projectionMissing) return { ok: true, recovered: false, reason, beforeUrl, afterUrl: currentPageUrl(), liveness, pageRole: "control", pageId, pageEpoch: controlPageEpoch, valuesRedacted: true };
|
||||||
const target = controlPageRecoveryTarget(liveness.snapshot, beforeUrl);
|
const target = controlPageRecoveryTarget(liveness.snapshot, beforeUrl);
|
||||||
await appendJsonl(files.control, eventRecord("control-page-unresponsive-before-command", {
|
await appendJsonl(files.control, eventRecord(projectionMissing ? "control-page-projection-missing-before-command" : "control-page-unresponsive-before-command", {
|
||||||
reason,
|
reason,
|
||||||
beforeUrl,
|
beforeUrl,
|
||||||
target,
|
target,
|
||||||
@@ -1714,7 +1801,15 @@ async function ensureControlPageResponsiveForCommand(reason) {
|
|||||||
pageEpoch: controlPageEpoch,
|
pageEpoch: controlPageEpoch,
|
||||||
valuesRedacted: true
|
valuesRedacted: true
|
||||||
}));
|
}));
|
||||||
const recovery = await recoverControlPageToTarget(reason, beforeUrl, target, liveness);
|
const promotion = await promoteObserverPageToControlForCommand(reason + "-observer-promotion", target, liveness);
|
||||||
|
await appendJsonl(files.control, eventRecord(promotion.ok ? "control-page-promoted-from-observer-before-command" : "control-page-observer-promotion-skipped-before-command", promotion));
|
||||||
|
if (promotion.ok === true) return promotion;
|
||||||
|
const recovery = await recoverControlPageToTarget(reason, beforeUrl, target, liveness, {
|
||||||
|
maxAttempts: 1,
|
||||||
|
navigation: { maxAttempts: 1, navigationTimeoutMs: 8000, readinessTimeoutMs: 4000, settleMs: 250, lateReadinessTimeoutMs: 1000 },
|
||||||
|
hydrationTimeoutMs: 4000,
|
||||||
|
hydrationHardTimeoutMs: 5000,
|
||||||
|
});
|
||||||
await appendJsonl(files.control, eventRecord(recovery.ok ? "control-page-recovered-before-command" : "control-page-recovery-failed-before-command", recovery));
|
await appendJsonl(files.control, eventRecord(recovery.ok ? "control-page-recovered-before-command" : "control-page-recovery-failed-before-command", recovery));
|
||||||
if (!recovery.ok) {
|
if (!recovery.ok) {
|
||||||
const error = new Error("control page recovery failed before " + reason);
|
const error = new Error("control page recovery failed before " + reason);
|
||||||
@@ -1738,7 +1833,15 @@ async function forceRecoverControlPageForCommand(reason) {
|
|||||||
pageEpoch: controlPageEpoch,
|
pageEpoch: controlPageEpoch,
|
||||||
valuesRedacted: true
|
valuesRedacted: true
|
||||||
}));
|
}));
|
||||||
const recovery = await recoverControlPageToTarget(reason, beforeUrl, target, liveness);
|
const promotion = await promoteObserverPageToControlForCommand(reason + "-observer-promotion", target, liveness);
|
||||||
|
await appendJsonl(files.control, eventRecord(promotion.ok ? "control-page-forced-promoted-from-observer-before-command" : "control-page-forced-observer-promotion-skipped-before-command", promotion));
|
||||||
|
if (promotion.ok === true) return promotion;
|
||||||
|
const recovery = await recoverControlPageToTarget(reason, beforeUrl, target, liveness, {
|
||||||
|
maxAttempts: 1,
|
||||||
|
navigation: { maxAttempts: 1, navigationTimeoutMs: 8000, readinessTimeoutMs: 4000, settleMs: 250, lateReadinessTimeoutMs: 1000 },
|
||||||
|
hydrationTimeoutMs: 4000,
|
||||||
|
hydrationHardTimeoutMs: 5000,
|
||||||
|
});
|
||||||
await appendJsonl(files.control, eventRecord(recovery.ok ? "control-page-forced-recovered-before-command" : "control-page-forced-recovery-failed-before-command", recovery));
|
await appendJsonl(files.control, eventRecord(recovery.ok ? "control-page-forced-recovered-before-command" : "control-page-forced-recovery-failed-before-command", recovery));
|
||||||
return recovery;
|
return recovery;
|
||||||
}
|
}
|
||||||
@@ -1758,7 +1861,7 @@ async function sendPrompt(text, options = {}) {
|
|||||||
? primaryEditor
|
? primaryEditor
|
||||||
: page.locator('textarea, [role="textbox"], [contenteditable="true"], input[type="text"]').last();
|
: page.locator('textarea, [role="textbox"], [contenteditable="true"], input[type="text"]').last();
|
||||||
try {
|
try {
|
||||||
await withHardTimeout(candidate.waitFor({ state: "visible", timeout: 15000 }), 20000, "sendPrompt composer editor did not become visible within 20s");
|
await withHardTimeout(candidate.waitFor({ state: "visible", timeout: 8000 }), 10000, "sendPrompt composer editor did not become visible within 10s");
|
||||||
editor = candidate;
|
editor = candidate;
|
||||||
break;
|
break;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user