fix(web-probe): expose auth retries and proxy public hwlab (#708)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
@@ -480,7 +480,6 @@ networkProfiles:
|
||||
- 192.168.0.0/16
|
||||
- 82.156.23.220
|
||||
- 74.48.78.17
|
||||
- hwlab.pikapython.com
|
||||
dockerBuildProxy:
|
||||
http: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808
|
||||
https: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808
|
||||
|
||||
@@ -7545,7 +7545,7 @@ function renderWebObserveStatusTable(value: Record<string, unknown>): string {
|
||||
: [];
|
||||
const next = record(value.next);
|
||||
const heartbeatError = record(heartbeat?.error) ?? record(manifest?.error);
|
||||
const heartbeatAuth = record(heartbeatError?.auth);
|
||||
const heartbeatAuth = record(heartbeat?.auth) ?? record(heartbeatError?.auth);
|
||||
const manifestNetwork = record(manifest?.network);
|
||||
const manifestBrowser = record(manifestNetwork?.browser);
|
||||
const manifestProxy = record(manifestNetwork?.proxy);
|
||||
@@ -7585,6 +7585,20 @@ function renderWebObserveStatusTable(value: Record<string, unknown>): string {
|
||||
]]),
|
||||
"",
|
||||
] : []),
|
||||
...(heartbeatAuth !== null ? [
|
||||
"Auth progress:",
|
||||
webObserveTable(["PHASE", "RETRY", "DELAY_MS", "STATUS", "RETRYABLE", "COOKIE", "EXHAUSTED", "LAST_ERROR"], [[
|
||||
webObserveText(heartbeatAuth.phase),
|
||||
webObserveText(heartbeatAuth.lastRetryLabel),
|
||||
webObserveText(heartbeatAuth.retryDelayMs),
|
||||
webObserveText(heartbeatAuth.lastStatusText === undefined ? heartbeatAuth.lastStatus : `${webObserveText(heartbeatAuth.lastStatus)} ${webObserveText(heartbeatAuth.lastStatusText)}`),
|
||||
webObserveText(heartbeatAuth.retryable),
|
||||
webObserveText(heartbeatAuth.cookiePresent),
|
||||
webObserveText(heartbeatAuth.retryExhausted),
|
||||
webObserveShort(webObserveText(heartbeatAuth.lastError), 80),
|
||||
]]),
|
||||
"",
|
||||
] : []),
|
||||
...(activeControl !== null ? [
|
||||
"Active command:",
|
||||
webObserveTable(["TYPE", "COMMAND", "AGE_S", "STARTED_AT", "DETAIL"], [[
|
||||
@@ -9294,7 +9308,8 @@ const readJson=(name)=>{try{return JSON.parse(fs.readFileSync(path.join(dir,name
|
||||
const tailJsonl=(name)=>{try{const file=path.join(dir,name); const st=fs.statSync(file); const maxBytes=Math.min(st.size,8*1024*1024); const fd=fs.openSync(file,'r'); try{const buf=Buffer.alloc(maxBytes); fs.readSync(fd,buf,0,maxBytes,st.size-maxBytes); const lines=buf.toString('utf8').split(/\\r?\\n/).filter(Boolean); if(st.size>maxBytes&&lines.length>0) lines.shift(); return lines.slice(-tailN).map(line=>{try{return JSON.parse(line)}catch{return {parseError:true, rawTail:line.slice(-500)}}});}finally{fs.closeSync(fd)}}catch{return []}};
|
||||
const short=(value)=>String(value||'').slice(0,160);
|
||||
const compactManifest=(item)=>item?{jobId:item.jobId,status:item.status,specRef:item.specRef,baseUrl:item.baseUrl,targetPath:item.targetPath,network:item.network,pageAuthority:item.pageAuthority,sampling:item.sampling,safety:item.safety,startedAt:item.startedAt,completedAt:item.completedAt,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null;
|
||||
const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,error:item.error?{message:short(item.error.message),auth:item.error.auth?{lastRetryLabel:item.error.auth.lastRetryLabel||null,retryExhausted:item.error.auth.retryExhausted===true,lastError:short(item.error.auth.lastError||'')}:null}:null}:null;
|
||||
const compactAuth=(auth)=>auth?{phase:auth.phase||null,lastRetryLabel:auth.lastRetryLabel||null,retryAttempt:auth.retryAttempt??null,retryMaxAttempts:auth.retryMaxAttempts??null,retryDelayMs:auth.retryDelayMs??null,lastStatus:auth.lastStatus??null,lastStatusText:auth.lastStatusText||null,retryable:auth.retryable??null,cookiePresent:auth.cookiePresent??null,retryExhausted:auth.retryExhausted===true,lastError:short(auth.lastError||'')}:null;
|
||||
const compactHeartbeat=(item)=>item?{ok:item.ok,jobId:item.jobId,pid:item.pid,stateDir:item.stateDir,status:item.status,pageId:item.pageId,baseUrl:item.baseUrl,currentUrl:item.currentUrl,sampleSeq:item.sampleSeq,commandSeq:item.commandSeq,activeCommandId:item.activeCommandId,auth:compactAuth(item.auth),updatedAt:item.updatedAt,uptimeMs:item.uptimeMs,error:item.error?{message:short(item.error.message),auth:compactAuth(item.error.auth)}:null}:null;
|
||||
const retryLabel=(detail)=>detail&&detail.auth?detail.auth.lastRetryLabel||'':detail&&detail.result?detail.result.lastRetryLabel||'':detail&&detail.error&&detail.error.auth?detail.error.auth.lastRetryLabel||'':'';
|
||||
const detailText=(detail)=>detail&&detail.error?short((detail.error.message||'')+(detail.error.auth&&detail.error.auth.lastError?' '+detail.error.auth.lastError:'')):detail&&detail.result?short([detail.result.statusText,detail.result.retryExhausted?'retry-exhausted':''].filter(Boolean).join(' ')):'';
|
||||
const compactControl=(item)=>({ts:item.ts,seq:item.seq,phase:item.phase,type:item.type,commandId:item.commandId,durationMs:item.detail&&item.detail.durationMs||null,retry:retryLabel(item.detail),detail:detailText(item.detail)});
|
||||
|
||||
@@ -406,6 +406,8 @@ async function authenticate(browserContext) {
|
||||
const maxDelayMs = 5000;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
const retryDelayMs = attempt < maxAttempts ? Math.min(maxDelayMs, initialDelayMs * (2 ** (attempt - 1))) : 0;
|
||||
const retryLabel = attempt + "/" + maxAttempts;
|
||||
await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", retryAttempt: attempt, retryMaxAttempts: maxAttempts, lastRetryLabel: retryLabel, retryDelayMs: 0, retryExhausted: false, valuesRedacted: true } }).catch(() => {});
|
||||
try {
|
||||
const response = await browserContext.request.post(loginUrl, {
|
||||
data: { username, password },
|
||||
@@ -418,7 +420,7 @@ async function authenticate(browserContext) {
|
||||
attempt,
|
||||
retryAttempt: attempt,
|
||||
retryMaxAttempts: maxAttempts,
|
||||
retryLabel: attempt + "/" + maxAttempts,
|
||||
retryLabel,
|
||||
retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0,
|
||||
method: "api",
|
||||
status: response.status(),
|
||||
@@ -429,6 +431,7 @@ async function authenticate(browserContext) {
|
||||
valuesRedacted: true,
|
||||
};
|
||||
attempts.push(item);
|
||||
await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item.retryLabel, retryAttempt: item.retryAttempt, retryMaxAttempts: item.retryMaxAttempts, retryDelayMs: item.retryDelayMs, lastStatus: item.status, lastStatusText: item.statusText, retryable: item.retryable, cookiePresent: item.cookiePresent, retryExhausted: false, valuesRedacted: true } }).catch(() => {});
|
||||
if (response.ok() && cookieState.cookiePresent) {
|
||||
return {
|
||||
ok: true,
|
||||
@@ -454,7 +457,7 @@ async function authenticate(browserContext) {
|
||||
attempt,
|
||||
retryAttempt: attempt,
|
||||
retryMaxAttempts: maxAttempts,
|
||||
retryLabel: attempt + "/" + maxAttempts,
|
||||
retryLabel,
|
||||
retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0,
|
||||
method: "api",
|
||||
status: 0,
|
||||
@@ -465,6 +468,8 @@ async function authenticate(browserContext) {
|
||||
cookieNames: [],
|
||||
valuesRedacted: true,
|
||||
});
|
||||
const item = attempts[attempts.length - 1] || null;
|
||||
await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item?.retryLabel || retryLabel, retryAttempt: attempt, retryMaxAttempts: maxAttempts, retryDelayMs: item?.retryDelayMs ?? 0, lastStatus: item?.status ?? 0, lastStatusText: item?.statusText ?? "request-error", retryable, cookiePresent: false, retryExhausted: false, lastError: item?.error || null, valuesRedacted: true } }).catch(() => {});
|
||||
if (!retryable) break;
|
||||
}
|
||||
if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs);
|
||||
@@ -489,6 +494,7 @@ async function authenticate(browserContext) {
|
||||
lastError: last?.error || null,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: failure.lastRetryLabel, retryAttempt: attempts.length, retryMaxAttempts: maxAttempts, retryDelayMs: 0, lastStatus: failure.status, lastStatusText: failure.statusText, retryable: failure.retryable, cookiePresent: failure.cookiePresent, retryExhausted: failure.retryExhausted, lastError: failure.lastError, valuesRedacted: true } }).catch(() => {});
|
||||
const error = new Error(authFailureMessage(failure));
|
||||
error.webProbeAuth = failure;
|
||||
throw error;
|
||||
|
||||
Reference in New Issue
Block a user