fix(web-probe): expose auth retries and proxy public hwlab (#708)

Co-authored-by: Codex <codex@noreply.local>

config/hwlab-node-lanes.yaml

@@ -480,7 +480,6 @@ networkProfiles:
         - 192.168.0.0/16
         - 82.156.23.220
         - 74.48.78.17
-        - hwlab.pikapython.com
     dockerBuildProxy:
       http: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808
       https: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808

scripts/src/hwlab-node-impl.ts

@@ -7545,7 +7545,7 @@ function renderWebObserveStatusTable(value: Record<string, unknown>): string {
     : [];
   const next = record(value.next);
   const heartbeatError = record(heartbeat?.error) ?? record(manifest?.error);
-  const heartbeatAuth = record(heartbeatError?.auth);
+  const heartbeatAuth = record(heartbeat?.auth) ?? record(heartbeatError?.auth);
   const manifestNetwork = record(manifest?.network);
   const manifestBrowser = record(manifestNetwork?.browser);
   const manifestProxy = record(manifestNetwork?.proxy);
@@ -7585,6 +7585,20 @@ function renderWebObserveStatusTable(value: Record<string, unknown>): string {
       ]]),
       "",
     ] : []),
+    ...(heartbeatAuth !== null ? [
+      "Auth progress:",
+      webObserveTable(["PHASE", "RETRY", "DELAY_MS", "STATUS", "RETRYABLE", "COOKIE", "EXHAUSTED", "LAST_ERROR"], [[
+        webObserveText(heartbeatAuth.phase),
+        webObserveText(heartbeatAuth.lastRetryLabel),
+        webObserveText(heartbeatAuth.retryDelayMs),
+        webObserveText(heartbeatAuth.lastStatusText === undefined ? heartbeatAuth.lastStatus : `${webObserveText(heartbeatAuth.lastStatus)} ${webObserveText(heartbeatAuth.lastStatusText)}`),
+        webObserveText(heartbeatAuth.retryable),
+        webObserveText(heartbeatAuth.cookiePresent),
+        webObserveText(heartbeatAuth.retryExhausted),
+        webObserveShort(webObserveText(heartbeatAuth.lastError), 80),
+      ]]),
+      "",
+    ] : []),
     ...(activeControl !== null ? [
       "Active command:",
       webObserveTable(["TYPE", "COMMAND", "AGE_S", "STARTED_AT", "DETAIL"], [[
@@ -9294,7 +9308,8 @@ const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name
 const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}};
 const short=(value)=>String(value||'').slice(0,160);
 const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null;
-const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null;
+const compactAuth=(auth)=>auth?{phase:auth.phase||null,lastRetryLabel:auth.lastRetryLabel||null,retryAttempt:auth.retryAttempt??null,retryMaxAttempts:auth.retryMaxAttempts??null,retryDelayMs:auth.retryDelayMs??null,lastStatus:auth.lastStatus??null,lastStatusText:auth.lastStatusText||null,retryable:auth.retryable??null,cookiePresent:auth.cookiePresent??null,retryExhausted:auth.retryExhausted===true,lastError:short(auth.lastError||'')}:null;
+const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,error:item.error?{message:short(item.error.message),auth:compactAuth(item.error.auth)}:null}:null;
 const retryLabel=(detail)=>detail&&detail.auth?detail.auth.lastRetryLabel||'':detail&&detail.result?detail.result.lastRetryLabel||'':detail&&detail.error&&detail.error.auth?detail.error.auth.lastRetryLabel||'':'';
 const detailText=(detail)=>detail&&detail.error?short((detail.error.message||'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':''].filter(Boolean).join(' ')):'';
 const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,durationMs:item.detail&&item.detail.durationMs||null,retry:retryLabel(item.detail),detail:detailText(item.detail)});

scripts/src/hwlab-node-web-observe-runner-source.ts

@@ -406,6 +406,8 @@ async function authenticate(browserContext) {
   const maxDelayMs = 5000;
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     const retryDelayMs = attempt < maxAttempts ? Math.min(maxDelayMs, initialDelayMs * (2 ** (attempt - 1))) : 0;
+    const retryLabel = attempt + "/" + maxAttempts;
+    await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", retryAttempt: attempt, retryMaxAttempts: maxAttempts, lastRetryLabel: retryLabel, retryDelayMs: 0, retryExhausted: false, valuesRedacted: true } }).catch(() => {});
     try {
       const response = await browserContext.request.post(loginUrl, {
         data: { username, password },
@@ -418,7 +420,7 @@ async function authenticate(browserContext) {
         attempt,
         retryAttempt: attempt,
         retryMaxAttempts: maxAttempts,
-        retryLabel: attempt + "/" + maxAttempts,
+        retryLabel,
         retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0,
         method: "api",
         status: response.status(),
@@ -429,6 +431,7 @@ async function authenticate(browserContext) {
         valuesRedacted: true,
       };
       attempts.push(item);
+      await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item.retryLabel, retryAttempt: item.retryAttempt, retryMaxAttempts: item.retryMaxAttempts, retryDelayMs: item.retryDelayMs, lastStatus: item.status, lastStatusText: item.statusText, retryable: item.retryable, cookiePresent: item.cookiePresent, retryExhausted: false, valuesRedacted: true } }).catch(() => {});
       if (response.ok() && cookieState.cookiePresent) {
         return {
           ok: true,
@@ -454,7 +457,7 @@ async function authenticate(browserContext) {
         attempt,
         retryAttempt: attempt,
         retryMaxAttempts: maxAttempts,
-        retryLabel: attempt + "/" + maxAttempts,
+        retryLabel,
         retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0,
         method: "api",
         status: 0,
@@ -465,6 +468,8 @@ async function authenticate(browserContext) {
         cookieNames: [],
         valuesRedacted: true,
       });
+      const item = attempts[attempts.length - 1] || null;
+      await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item?.retryLabel || retryLabel, retryAttempt: attempt, retryMaxAttempts: maxAttempts, retryDelayMs: item?.retryDelayMs ?? 0, lastStatus: item?.status ?? 0, lastStatusText: item?.statusText ?? "request-error", retryable, cookiePresent: false, retryExhausted: false, lastError: item?.error || null, valuesRedacted: true } }).catch(() => {});
       if (!retryable) break;
     }
     if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs);
@@ -489,6 +494,7 @@ async function authenticate(browserContext) {
     lastError: last?.error || null,
     valuesRedacted: true,
   };
+  await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: failure.lastRetryLabel, retryAttempt: attempts.length, retryMaxAttempts: maxAttempts, retryDelayMs: 0, lastStatus: failure.status, lastStatusText: failure.statusText, retryable: failure.retryable, cookiePresent: failure.cookiePresent, retryExhausted: failure.retryExhausted, lastError: failure.lastError, valuesRedacted: true } }).catch(() => {});
   const error = new Error(authFailureMessage(failure));
   error.webProbeAuth = failure;
   throw error;