From 457cbd1e38e09b0adb1bbf99f551f2692d2d2ea1 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 2 Jul 2026 07:37:54 +0000 Subject: [PATCH] fix: surface web probe target readiness blockers --- ...lab-node-web-observe-analyzer-io-source.ts | 54 +++++++++++++++++++ scripts/src/hwlab-node-web-observe-render.ts | 41 ++++++++++++++ ...-node-web-observe-runner-control-source.ts | 42 +++++++++++++-- .../hwlab-node-web-observe-runner-source.ts | 8 ++- scripts/src/hwlab-node/web-observe-scripts.ts | 8 +-- 5 files changed, 146 insertions(+), 7 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-analyzer-io-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-io-source.ts index 11ac94d3..4e05c0bc 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-io-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-io-source.ts @@ -68,6 +68,22 @@ async function readCommandBucket(dir, bucket) { function buildToolFindings({ manifest, heartbeat, commandState }) { const findings = []; const diagnostics = heartbeatDiagnostics(manifest, heartbeat); + const startupReadiness = toolStartupReadiness(manifest, heartbeat); + if (startupReadiness !== null) { + findings.push({ + id: "tool-target-page-not-ready", + severity: "red", + summary: "web-probe observe target page did not reach the expected Workbench app shell; treat this as probe setup/runtime access blocker before interpreting business turns or frontend hotspots", + count: 1, + reason: startupReadiness.reason, + rootCause: startupReadiness.reason === "nav-access-denied" ? "web_probe_target_nav_access_denied" : startupReadiness.reason === "workbench-blank-document" ? "web_probe_target_blank_workbench_document" : "web_probe_target_page_not_ready", + rootCauseStatus: "confirmed-from-startup-readiness", + rootCauseConfidence: "high", + nextAction: "Fix the selected node/lane Web access or Workbench shell loading first, then rerun observe start and performanceCapture; do not treat an empty performance report from this run as a business performance result.", + readiness: startupReadiness, + valuesRedacted: true + }); + } if (diagnostics.heartbeatStale) { findings.push({ id: "tool-runner-heartbeat-stale", @@ -112,6 +128,44 @@ function buildToolFindings({ manifest, heartbeat, commandState }) { return findings; } +function toolStartupReadiness(manifest, heartbeat) { + const candidates = [ + heartbeat?.error?.navigationReadiness, + heartbeat?.error?.details?.readiness, + heartbeat?.error?.details?.readinessAfterWait, + heartbeat?.error?.details?.readinessBeforeClick, + manifest?.error?.navigationReadiness, + manifest?.error?.details?.readiness, + manifest?.error?.details?.readinessAfterWait, + manifest?.error?.details?.readinessBeforeClick, + ]; + for (const candidate of candidates) { + const readiness = candidate && typeof candidate === "object" && !Array.isArray(candidate) ? candidate : null; + if (!readiness) continue; + const snapshot = readiness.snapshot && typeof readiness.snapshot === "object" && !Array.isArray(readiness.snapshot) ? readiness.snapshot : readiness; + const reason = typeof readiness.reason === "string" && readiness.reason ? readiness.reason : typeof snapshot.reason === "string" && snapshot.reason ? snapshot.reason : null; + if (reason === null) continue; + return { + reason, + path: typeof snapshot.path === "string" ? snapshot.path : null, + search: typeof snapshot.search === "string" ? snapshot.search : null, + blockedReason: typeof snapshot.blockedReason === "string" ? snapshot.blockedReason : null, + readyState: typeof snapshot.readyState === "string" ? snapshot.readyState : null, + workbenchShellVisible: snapshot.workbenchShellVisible === true, + sessionCreatePresent: snapshot.sessionCreatePresent === true, + sessionCreateVisible: snapshot.sessionCreateVisible === true, + sessionRailPresent: snapshot.sessionRailPresent === true, + commandInputPresent: snapshot.commandInputPresent === true, + activeTabPresent: snapshot.activeTabPresent === true, + loginVisible: snapshot.loginVisible === true, + bodyTextBytes: Number.isFinite(Number(snapshot.bodyTextBytes)) ? Number(snapshot.bodyTextBytes) : null, + bodyTextHash: typeof snapshot.bodyTextHash === "string" ? snapshot.bodyTextHash : null, + valuesRedacted: true + }; + } + return null; +} + function heartbeatDiagnostics(manifest, heartbeat) { const status = String(heartbeat?.status || manifest?.status || ""); const terminal = /^(completed|failed|force-stopped|stopped|abandoned)$/u.test(status); diff --git a/scripts/src/hwlab-node-web-observe-render.ts b/scripts/src/hwlab-node-web-observe-render.ts index d47f9ab5..5329aa73 100644 --- a/scripts/src/hwlab-node-web-observe-render.ts +++ b/scripts/src/hwlab-node-web-observe-render.ts @@ -56,6 +56,7 @@ function renderWebObserveStatusTable(value: Record): string { const next = record(value.next); const heartbeatError = record(heartbeat?.error) ?? record(manifest?.error); const heartbeatAuth = record(heartbeat?.auth) ?? record(heartbeatError?.auth); + const heartbeatReadiness = record(heartbeatError?.readiness); const manifestNetwork = record(manifest?.network); const manifestBrowser = record(manifestNetwork?.browser); const manifestProxy = record(manifestNetwork?.proxy); @@ -114,6 +115,24 @@ function renderWebObserveStatusTable(value: Record): string { ]]), "", ] : []), + ...(heartbeatReadiness !== null ? [ + "Target readiness:", + webObserveTable(["REASON", "PATH", "BLOCKED", "READY", "SHELL", "CREATE", "RAIL", "INPUT", "TAB", "LOGIN", "BODY_BYTES", "BODY_HASH"], [[ + heartbeatReadiness.reason, + webObserveShort(webObserveText(heartbeatReadiness.path), 40), + heartbeatReadiness.blockedReason, + heartbeatReadiness.readyState, + heartbeatReadiness.workbenchShellVisible, + heartbeatReadiness.sessionCreateVisible === true ? "visible" : heartbeatReadiness.sessionCreatePresent === true ? "hidden" : "missing", + heartbeatReadiness.sessionRailCollapsed === true ? "collapsed" : heartbeatReadiness.sessionRailPresent === true ? "present" : "missing", + heartbeatReadiness.commandInputPresent, + heartbeatReadiness.activeTabPresent, + heartbeatReadiness.loginVisible, + heartbeatReadiness.bodyTextBytes, + webObserveShort(webObserveText(heartbeatReadiness.bodyTextHash), 24), + ]]), + "", + ] : []), ...(heartbeatAuth !== null ? [ "Auth progress:", webObserveTable(["PHASE", "RETRY", "DELAY_MS", "STATUS", "RETRYABLE", "COOKIE", "EXHAUSTED", "LAST_ERROR"], [[ @@ -187,6 +206,10 @@ function renderWebObserveCommandTable(value: Record): string { const observerCommand = record(value.observerCommand); const observer = record(value.observer); const result = record(observer?.result); + const error = record(observer?.error); + const details = record(error?.details); + const rawReadiness = record(error?.navigationReadiness) ?? record(details?.readiness) ?? record(details?.readinessAfterWait) ?? record(details?.readinessBeforeClick); + const readiness = record(rawReadiness?.snapshot) ?? rawReadiness; const id = webObserveText(value.id); const full = value.full === true; const lines = [ @@ -202,6 +225,24 @@ function renderWebObserveCommandTable(value: Record): string { webObserveShort(webObserveText(observerCommand?.textHash), 24), webObserveShort(webObserveText(result?.mark ?? result?.currentUrl ?? observer?.error ?? observer?.queued), 80), ]]), + ...(readiness !== null ? [ + "", + "Target readiness:", + webObserveTable(["REASON", "PATH", "BLOCKED", "READY", "SHELL", "CREATE", "RAIL", "INPUT", "TAB", "LOGIN", "BODY_BYTES", "BODY_HASH"], [[ + rawReadiness?.reason ?? readiness.reason, + webObserveShort(webObserveText(readiness.path), 40), + readiness.blockedReason, + readiness.readyState, + readiness.workbenchShellVisible, + readiness.sessionCreateVisible === true ? "visible" : readiness.sessionCreatePresent === true ? "hidden" : "missing", + readiness.sessionRailCollapsed === true ? "collapsed" : readiness.sessionRailPresent === true ? "present" : "missing", + readiness.commandInputPresent, + readiness.activeTabPresent, + readiness.loginVisible, + readiness.bodyTextBytes, + webObserveShort(webObserveText(readiness.bodyTextHash), 24), + ]]), + ] : []), ...(full ? [ "", "Full command result:", diff --git a/scripts/src/hwlab-node-web-observe-runner-control-source.ts b/scripts/src/hwlab-node-web-observe-runner-control-source.ts index fe76261f..74bacd4e 100644 --- a/scripts/src/hwlab-node-web-observe-runner-control-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-control-source.ts @@ -785,7 +785,15 @@ async function gotoTarget(rawTarget, options = {}) { if (!readiness.ok) { const pageProvenance = await refreshPageProvenance("goto-degraded", httpStatus).catch(() => null); attempts.push({ attempt, ok: false, degraded: true, httpStatus, readiness, failureKind: readiness.reason || "workbench-app-not-ready" }); - return { beforeUrl, afterUrl: currentPageUrl(), httpStatus, pageId, degraded: true, degradedReason: readiness.reason || "workbench-app-not-ready", pageProvenance: compactPageProvenance(pageProvenance), readiness, attempts }; + if (!readinessRetryable(readiness)) { + return { beforeUrl, afterUrl: currentPageUrl(), httpStatus, pageId, degraded: true, degradedReason: readiness.reason || "workbench-app-not-ready", pageProvenance: compactPageProvenance(pageProvenance), readiness, attempts }; + } + if (attempt >= maxAttempts) { + return { beforeUrl, afterUrl: currentPageUrl(), httpStatus, pageId, degraded: true, degradedReason: readiness.reason || "workbench-app-not-ready", pageProvenance: compactPageProvenance(pageProvenance), readiness, attempts }; + } + await recreateControlPageForNavigation("retryable-readiness-" + (readiness.reason || "target-not-ready"), attempt).catch((resetError) => appendJsonl(files.errors, eventRecord("navigation-page-reset-error", { commandId: activeCommandId, attempt, error: errorSummary(resetError) }))); + await page.waitForTimeout(1500 * attempt).catch(() => {}); + continue; } const pageProvenance = await refreshPageProvenance("goto", httpStatus); attempts.push({ attempt, ok: true, httpStatus, readiness }); @@ -931,6 +939,11 @@ function isRetryableNavigationError(message) { return /net::ERR_NETWORK_CHANGED|net::ERR_ABORTED|net::ERR_CONNECTION_RESET|net::ERR_NAME_NOT_RESOLVED|Navigation timeout|page\.goto:\s*timeout|timeout\s+\d+ms\s+exceeded|workbench-app-not-ready/iu.test(String(message || "")); } +function readinessRetryable(readiness) { + const reason = String(readiness?.reason || ""); + return !/nav-access-denied|workbench-blank-document|login-visible/iu.test(reason); +} + function navigationFailureKind(message) { const text = String(message || ""); if (/net::ERR_NETWORK_CHANGED/iu.test(text)) return "net::ERR_NETWORK_CHANGED"; @@ -990,9 +1003,10 @@ async function waitForTargetPageReady(targetPage, targetUrl, options = {}) { }, null, { timeout: timeoutMs }).catch(() => null); const snapshot = await workbenchReadinessSnapshot(targetPage); const ok = snapshot.workbenchShellVisible === true; + const reason = workbenchReadinessReason(snapshot, ok); return { ok, - reason: ok ? "workbench-ready" : snapshot.loginVisible ? "login-visible" : "workbench-app-not-ready", + reason, durationMs: Date.now() - started, snapshot, valuesRedacted: true @@ -1010,9 +1024,13 @@ async function workbenchReadinessSnapshot(targetPage) { const sessionCreate = document.querySelector("#session-create"); const sessionRail = document.querySelector("#session-sidebar"); const sessionCollapseToggle = document.querySelector("#session-collapse-toggle"); + const params = new URLSearchParams(window.location.search || ""); + const bodyText = String(document.body?.innerText || ""); return { url: window.location.href, path: window.location.pathname, + search: window.location.search || "", + blockedReason: params.get("blocked"), readyState: document.readyState, workbenchShellVisible: visible(document.querySelector("#workspace, .workbench-route")), sessionCreatePresent: Boolean(sessionCreate), @@ -1026,7 +1044,8 @@ async function workbenchReadinessSnapshot(targetPage) { activeTabPresent: visible(document.querySelector(".session-tab[data-active='true'], .session-tab[aria-selected='true']")), warningPresent: visible(document.querySelector(".composer-warning")), loginVisible: visible(document.querySelector("form.login-card, .login-card, [data-testid='login']")), - bodyTextPreview: String(document.body?.innerText || "").slice(0, 2000), + bodyTextBytes: new TextEncoder().encode(bodyText).length, + bodyTextPreview: bodyText.slice(0, 2000), valuesRedacted: true }; }).catch((error) => ({ error: errorSummary(error), valuesRedacted: true })); @@ -1037,6 +1056,23 @@ async function workbenchReadinessSnapshot(targetPage) { return snapshot; } +function workbenchReadinessReason(snapshot, ok = false) { + if (ok) return "workbench-ready"; + const blocked = String(snapshot?.blockedReason || ""); + if (blocked === "nav_access_denied") return "nav-access-denied"; + if (snapshot?.loginVisible === true) return "login-visible"; + if ( + snapshot?.readyState === "complete" + && snapshot?.workbenchShellVisible === false + && snapshot?.sessionRailPresent === false + && snapshot?.commandInputPresent === false + && (snapshot?.bodyTextBytes === 0 || snapshot?.bodyTextHash === "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + ) { + return "workbench-blank-document"; + } + return "workbench-app-not-ready"; +} + async function projectManagementReadinessSnapshot(targetPage) { const selectors = projectManagement.readinessSelectors; return targetPage.evaluate((input) => { diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 6cce66f5..c5bfdecc 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -111,7 +111,13 @@ try { page = await context.newPage(); attachPassiveListeners(page, "control", pageId); auth = await runControlCommand({ id: "startup-login", type: "login", createdAt: startedAt, source: "startup" }, async () => authenticate(context)); - await runControlCommand({ id: "startup-goto", type: "goto", path: targetPath, createdAt: new Date().toISOString(), source: "startup" }, async () => gotoTarget(targetPath)); + const startupGoto = await runControlCommand({ id: "startup-goto", type: "goto", path: targetPath, createdAt: new Date().toISOString(), source: "startup" }, async () => gotoTarget(targetPath)); + if (startupGoto?.degraded === true) { + const error = new Error("startup target page is not ready: " + (startupGoto.degradedReason || "target-not-ready")); + error.details = startupGoto; + error.navigationReadiness = startupGoto.readiness || null; + throw error; + } observerPage = await context.newPage(); attachPassiveListeners(observerPage, "observer", observerPageId); await runControlCommand({ id: "startup-observer-goto", type: "observerGoto", path: targetPath, createdAt: new Date().toISOString(), source: "startup" }, async () => { diff --git a/scripts/src/hwlab-node/web-observe-scripts.ts b/scripts/src/hwlab-node/web-observe-scripts.ts index a92ed96a..f604e4e0 100644 --- a/scripts/src/hwlab-node/web-observe-scripts.ts +++ b/scripts/src/hwlab-node/web-observe-scripts.ts @@ -46,11 +46,13 @@ export function nodeWebObserveStatusNodeScript(tailLines: number, node: string, const readJsonPath=(file)=>{try{return JSON.parse(fs.readFileSync(file,'utf8'))}catch{return null}}; const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}}; const short=(value)=>String(value||'').slice(0,160); - const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null; + const compactReadiness=(value)=>{const raw=value&&typeof value==='object'?value:null; const snapshot=raw&&raw.snapshot&&typeof raw.snapshot==='object'?raw.snapshot:raw; return raw?{reason:short(raw.reason||snapshot.reason,80),path:short(snapshot.path,120),search:short(snapshot.search,120),blockedReason:short(snapshot.blockedReason,80),readyState:short(snapshot.readyState,24),workbenchShellVisible:snapshot.workbenchShellVisible===true,sessionCreatePresent:snapshot.sessionCreatePresent===true,sessionCreateVisible:snapshot.sessionCreateVisible===true,sessionRailPresent:snapshot.sessionRailPresent===true,sessionRailCollapsed:snapshot.sessionRailCollapsed??null,commandInputPresent:snapshot.commandInputPresent===true,activeTabPresent:snapshot.activeTabPresent===true,loginVisible:snapshot.loginVisible===true,bodyTextBytes:snapshot.bodyTextBytes??null,bodyTextHash:short(snapshot.bodyTextHash,80),valuesRedacted:true}:null}; + const errorReadiness=(error)=>{const details=error&&error.details&&typeof error.details==='object'?error.details:null; return compactReadiness((error&&error.navigationReadiness)||details&&details.readiness||details&&details.readinessAfterWait||details&&details.readinessBeforeClick)}; + const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),readiness:errorReadiness(item.error),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null; const compactAuth=(auth)=>auth?{phase:auth.phase||null,lastRetryLabel:auth.lastRetryLabel||null,retryAttempt:auth.retryAttempt??null,retryMaxAttempts:auth.retryMaxAttempts??null,retryDelayMs:auth.retryDelayMs??null,lastStatus:auth.lastStatus??null,lastStatusText:auth.lastStatusText||null,retryable:auth.retryable??null,cookiePresent:auth.cookiePresent??null,retryExhausted:auth.retryExhausted===true,lastError:short(auth.lastError||'')}:null; - const compactHeartbeat=(item,diag)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,ageSeconds:diag.heartbeatAgeSeconds,stale:diag.heartbeatStale,staleAfterSeconds:diag.heartbeatStaleAfterSeconds,effectiveLiveness:diag.effectiveLiveness,error:item.error?{message:short(item.error.message),auth:compactAuth(item.error.auth)}:null}:null; + const compactHeartbeat=(item,diag)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,ageSeconds:diag.heartbeatAgeSeconds,stale:diag.heartbeatStale,staleAfterSeconds:diag.heartbeatStaleAfterSeconds,effectiveLiveness:diag.effectiveLiveness,error:item.error?{message:short(item.error.message),readiness:errorReadiness(item.error),auth:compactAuth(item.error.auth)}:null}:null; const retryLabel=(detail)=>detail&&detail.auth?detail.auth.lastRetryLabel||'':detail&&detail.result?detail.result.lastRetryLabel||'':detail&&detail.error&&detail.error.auth?detail.error.auth.lastRetryLabel||'':''; - const detailText=(detail)=>detail&&detail.error?short((detail.error.message||'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':''].filter(Boolean).join(' ')):''; + const detailText=(detail)=>{const readiness=detail&&detail.error?errorReadiness(detail.error):null; return detail&&detail.error?short((detail.error.message||'')+(readiness&&readiness.reason?' readiness='+readiness.reason:'')+(readiness&&readiness.blockedReason?' blocked='+readiness.blockedReason:'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':'',detail.result.degradedReason?'degraded='+detail.result.degradedReason:''].filter(Boolean).join(' ')):''}; const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,durationMs:item.detail&&item.detail.durationMs||null,retry:retryLabel(item.detail),detail:detailText(item.detail)}); const compactSample=(item)=>({seq:item.seq,ts:item.ts,path:item.path,routeSessionId:item.routeSessionId||null,activeSessionId:item.activeSessionId||null,messageCount:Array.isArray(item.messages)?item.messages.length:0,traceRowCount:Array.isArray(item.traceRows)?item.traceRows.length:0,loadingCount:Array.isArray(item.loadings)?item.loadings.length:0,loadingOwners:Array.isArray(item.loadings)?Array.from(new Set(item.loadings.map((loading)=>loading.ownerLabel||loading.ownerKind||loading.ownerKey||'loading'))).slice(0,4):[],sessionRail:item.sessionRail?{visibleCount:item.sessionRail.visibleCount??null,fallbackTitleCount:item.sessionRail.fallbackTitleCount??null,fallbackTitleRatio:item.sessionRail.fallbackTitleRatio??null,fallbackItems:Array.isArray(item.sessionRail.fallbackItems)?item.sessionRail.fallbackItems.slice(0,4).map((fallback)=>({sessionIdPrefix:fallback.sessionIdPrefix??null,titlePreview:fallback.titlePreview??null,active:fallback.active===true})):[]}:null}); const compactNetwork=(item)=>({ts:item.ts,type:item.type,method:item.method,url:short(item.url),status:item.status||null,failure:item.failure?short(item.failure):null});