diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 6ce981d5..33287f75 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -480,7 +480,6 @@ networkProfiles: - 192.168.0.0/16 - 82.156.23.220 - 74.48.78.17 - - hwlab.pikapython.com dockerBuildProxy: http: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 https: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 4d1a21a4..55a409c2 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -7545,7 +7545,7 @@ function renderWebObserveStatusTable(value: Record): string { : []; const next = record(value.next); const heartbeatError = record(heartbeat?.error) ?? record(manifest?.error); - const heartbeatAuth = record(heartbeatError?.auth); + const heartbeatAuth = record(heartbeat?.auth) ?? record(heartbeatError?.auth); const manifestNetwork = record(manifest?.network); const manifestBrowser = record(manifestNetwork?.browser); const manifestProxy = record(manifestNetwork?.proxy); @@ -7585,6 +7585,20 @@ function renderWebObserveStatusTable(value: Record): string { ]]), "", ] : []), + ...(heartbeatAuth !== null ? [ + "Auth progress:", + webObserveTable(["PHASE", "RETRY", "DELAY_MS", "STATUS", "RETRYABLE", "COOKIE", "EXHAUSTED", "LAST_ERROR"], [[ + webObserveText(heartbeatAuth.phase), + webObserveText(heartbeatAuth.lastRetryLabel), + webObserveText(heartbeatAuth.retryDelayMs), + webObserveText(heartbeatAuth.lastStatusText === undefined ? heartbeatAuth.lastStatus : `${webObserveText(heartbeatAuth.lastStatus)} ${webObserveText(heartbeatAuth.lastStatusText)}`), + webObserveText(heartbeatAuth.retryable), + webObserveText(heartbeatAuth.cookiePresent), + webObserveText(heartbeatAuth.retryExhausted), + webObserveShort(webObserveText(heartbeatAuth.lastError), 80), + ]]), + "", + ] : []), ...(activeControl !== null ? [ "Active command:", webObserveTable(["TYPE", "COMMAND", "AGE_S", "STARTED_AT", "DETAIL"], [[ @@ -9294,7 +9308,8 @@ const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}}; const short=(value)=>String(value||'').slice(0,160); const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null; -const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null; +const compactAuth=(auth)=>auth?{phase:auth.phase||null,lastRetryLabel:auth.lastRetryLabel||null,retryAttempt:auth.retryAttempt??null,retryMaxAttempts:auth.retryMaxAttempts??null,retryDelayMs:auth.retryDelayMs??null,lastStatus:auth.lastStatus??null,lastStatusText:auth.lastStatusText||null,retryable:auth.retryable??null,cookiePresent:auth.cookiePresent??null,retryExhausted:auth.retryExhausted===true,lastError:short(auth.lastError||'')}:null; +const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,error:item.error?{message:short(item.error.message),auth:compactAuth(item.error.auth)}:null}:null; const retryLabel=(detail)=>detail&&detail.auth?detail.auth.lastRetryLabel||'':detail&&detail.result?detail.result.lastRetryLabel||'':detail&&detail.error&&detail.error.auth?detail.error.auth.lastRetryLabel||'':''; const detailText=(detail)=>detail&&detail.error?short((detail.error.message||'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':''].filter(Boolean).join(' ')):''; const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,durationMs:item.detail&&item.detail.durationMs||null,retry:retryLabel(item.detail),detail:detailText(item.detail)}); diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 711fbc9a..490739c1 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -406,6 +406,8 @@ async function authenticate(browserContext) { const maxDelayMs = 5000; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const retryDelayMs = attempt < maxAttempts ? Math.min(maxDelayMs, initialDelayMs * (2 ** (attempt - 1))) : 0; + const retryLabel = attempt + "/" + maxAttempts; + await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", retryAttempt: attempt, retryMaxAttempts: maxAttempts, lastRetryLabel: retryLabel, retryDelayMs: 0, retryExhausted: false, valuesRedacted: true } }).catch(() => {}); try { const response = await browserContext.request.post(loginUrl, { data: { username, password }, @@ -418,7 +420,7 @@ async function authenticate(browserContext) { attempt, retryAttempt: attempt, retryMaxAttempts: maxAttempts, - retryLabel: attempt + "/" + maxAttempts, + retryLabel, retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0, method: "api", status: response.status(), @@ -429,6 +431,7 @@ async function authenticate(browserContext) { valuesRedacted: true, }; attempts.push(item); + await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item.retryLabel, retryAttempt: item.retryAttempt, retryMaxAttempts: item.retryMaxAttempts, retryDelayMs: item.retryDelayMs, lastStatus: item.status, lastStatusText: item.statusText, retryable: item.retryable, cookiePresent: item.cookiePresent, retryExhausted: false, valuesRedacted: true } }).catch(() => {}); if (response.ok() && cookieState.cookiePresent) { return { ok: true, @@ -454,7 +457,7 @@ async function authenticate(browserContext) { attempt, retryAttempt: attempt, retryMaxAttempts: maxAttempts, - retryLabel: attempt + "/" + maxAttempts, + retryLabel, retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0, method: "api", status: 0, @@ -465,6 +468,8 @@ async function authenticate(browserContext) { cookieNames: [], valuesRedacted: true, }); + const item = attempts[attempts.length - 1] || null; + await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item?.retryLabel || retryLabel, retryAttempt: attempt, retryMaxAttempts: maxAttempts, retryDelayMs: item?.retryDelayMs ?? 0, lastStatus: item?.status ?? 0, lastStatusText: item?.statusText ?? "request-error", retryable, cookiePresent: false, retryExhausted: false, lastError: item?.error || null, valuesRedacted: true } }).catch(() => {}); if (!retryable) break; } if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs); @@ -489,6 +494,7 @@ async function authenticate(browserContext) { lastError: last?.error || null, valuesRedacted: true, }; + await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: failure.lastRetryLabel, retryAttempt: attempts.length, retryMaxAttempts: maxAttempts, retryDelayMs: 0, lastStatus: failure.status, lastStatusText: failure.statusText, retryable: failure.retryable, cookiePresent: failure.cookiePresent, retryExhausted: failure.retryExhausted, lastError: failure.lastError, valuesRedacted: true } }).catch(() => {}); const error = new Error(authFailureMessage(failure)); error.webProbeAuth = failure; throw error;