diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 22749014..dfe6a86a 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -74,9 +74,9 @@ try { const { chromium } = await launcher.importPlaywright(); browser = await launcher.launchChromium(chromium, chromiumLaunchOptions); context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) }); - auth = await runControlCommand({ id: "startup-login", type: "login", createdAt: startedAt, source: "startup" }, async () => authenticate(context)); page = await context.newPage(); attachPassiveListeners(page, "control", pageId); + auth = await runControlCommand({ id: "startup-login", type: "login", createdAt: startedAt, source: "startup" }, async () => authenticate(context, page)); await runControlCommand({ id: "startup-goto", type: "goto", path: targetPath, createdAt: new Date().toISOString(), source: "startup" }, async () => gotoTarget(targetPath)); observerPage = await context.newPage(); attachPassiveListeners(observerPage, "observer", observerPageId); @@ -295,7 +295,7 @@ async function processCommand(command) { await writeHeartbeat({ status: "running", activeCommandId }); await appendJsonl(files.control, controlRecord(command, "started", commandInputSummary(command))); switch (command.type) { - case "login": return authenticate(context); + case "login": return authenticate(context, page); case "preflight": return preflightSummary(); case "goto": return withObserverSync(await gotoTarget(command.path || command.url || targetPath), "goto"); case "newSession": return withObserverSync(await createSessionFromUi(), "newSession"); @@ -402,81 +402,87 @@ async function runControlCommand(command, fn) { } } -async function authenticate(browserContext) { +async function authenticate(browserContext, authPage) { const loginUrl = new URL("/auth/login", baseUrl).toString(); + let activeAuthPage = authPage && !authPage.isClosed() ? authPage : null; + let closeAuthPage = false; + if (!activeAuthPage) { + activeAuthPage = await browserContext.newPage(); + closeAuthPage = true; + } const attempts = []; const maxAttempts = 5; const initialDelayMs = 250; const maxDelayMs = 5000; - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const retryDelayMs = attempt < maxAttempts ? Math.min(maxDelayMs, initialDelayMs * (2 ** (attempt - 1))) : 0; - const retryLabel = attempt + "/" + maxAttempts; - await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", retryAttempt: attempt, retryMaxAttempts: maxAttempts, lastRetryLabel: retryLabel, retryDelayMs: 0, retryExhausted: false, valuesRedacted: true } }).catch(() => {}); - try { - const response = await browserContext.request.post(loginUrl, { - data: { username, password }, - headers: { accept: "application/json", "content-type": "application/json" }, - timeout: 12000, - }); - const cookieState = await readAuthCookieState(browserContext); - const retryable = isRetryableAuthStatus(response.status()); - const item = { - attempt, - retryAttempt: attempt, - retryMaxAttempts: maxAttempts, - retryLabel, - retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0, - method: "api", - status: response.status(), - statusText: response.statusText(), - retryable, - cookiePresent: cookieState.cookiePresent, - cookieNames: cookieState.cookieNames, - valuesRedacted: true, - }; - attempts.push(item); - await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item.retryLabel, retryAttempt: item.retryAttempt, retryMaxAttempts: item.retryMaxAttempts, retryDelayMs: item.retryDelayMs, lastStatus: item.status, lastStatusText: item.statusText, retryable: item.retryable, cookiePresent: item.cookiePresent, retryExhausted: false, valuesRedacted: true } }).catch(() => {}); - if (response.ok() && cookieState.cookiePresent) { - return { - ok: true, - method: "api", - loginPath: new URL(loginUrl).pathname, - status: response.status(), - statusText: response.statusText(), - cookiePresent: true, - cookieNames: cookieState.cookieNames, - attempts, - retryCount: attempt - 1, + try { + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const retryDelayMs = attempt < maxAttempts ? Math.min(maxDelayMs, initialDelayMs * (2 ** (attempt - 1))) : 0; + const retryLabel = attempt + "/" + maxAttempts; + await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", retryAttempt: attempt, retryMaxAttempts: maxAttempts, lastRetryLabel: retryLabel, retryDelayMs: 0, retryExhausted: false, valuesRedacted: true } }).catch(() => {}); + try { + const response = await pageAuthLogin(activeAuthPage, loginUrl); + const cookieState = await readAuthCookieState(browserContext); + const retryable = isRetryableAuthStatus(response.status); + const item = { + attempt, + retryAttempt: attempt, retryMaxAttempts: maxAttempts, - lastRetryLabel: attempt + "/" + maxAttempts, - retryExhausted: false, - retryable: false, + retryLabel, + retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0, + method: "api", + status: response.status, + statusText: response.statusText, + retryable, + cookiePresent: cookieState.cookiePresent, + cookieNames: cookieState.cookieNames, valuesRedacted: true, }; + attempts.push(item); + await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item.retryLabel, retryAttempt: item.retryAttempt, retryMaxAttempts: item.retryMaxAttempts, retryDelayMs: item.retryDelayMs, lastStatus: item.status, lastStatusText: item.statusText, retryable: item.retryable, cookiePresent: item.cookiePresent, retryExhausted: false, valuesRedacted: true } }).catch(() => {}); + if (response.ok && cookieState.cookiePresent) { + return { + ok: true, + method: "api", + loginPath: new URL(loginUrl).pathname, + status: response.status, + statusText: response.statusText, + cookiePresent: true, + cookieNames: cookieState.cookieNames, + attempts, + retryCount: attempt - 1, + retryMaxAttempts: maxAttempts, + lastRetryLabel: attempt + "/" + maxAttempts, + retryExhausted: false, + retryable: false, + valuesRedacted: true, + }; + } + if (!retryable) break; + } catch (error) { + const retryable = isRetryableAuthError(error); + attempts.push({ + attempt, + retryAttempt: attempt, + retryMaxAttempts: maxAttempts, + retryLabel, + retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0, + method: "api", + status: 0, + statusText: "request-error", + retryable, + error: error && error.message ? truncate(error.message, 500) : truncate(String(error), 500), + cookiePresent: false, + cookieNames: [], + valuesRedacted: true, + }); + const item = attempts[attempts.length - 1] || null; + await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item?.retryLabel || retryLabel, retryAttempt: attempt, retryMaxAttempts: maxAttempts, retryDelayMs: item?.retryDelayMs ?? 0, lastStatus: item?.status ?? 0, lastStatusText: item?.statusText ?? "request-error", retryable, cookiePresent: false, retryExhausted: false, lastError: item?.error || null, valuesRedacted: true } }).catch(() => {}); + if (!retryable) break; } - if (!retryable) break; - } catch (error) { - const retryable = isRetryableAuthError(error); - attempts.push({ - attempt, - retryAttempt: attempt, - retryMaxAttempts: maxAttempts, - retryLabel, - retryDelayMs: retryable && attempt < maxAttempts ? retryDelayMs : 0, - method: "api", - status: 0, - statusText: "request-error", - retryable, - error: error && error.message ? truncate(error.message, 500) : truncate(String(error), 500), - cookiePresent: false, - cookieNames: [], - valuesRedacted: true, - }); - const item = attempts[attempts.length - 1] || null; - await writeHeartbeat({ status: terminalStatus, auth: { phase: "api-login", lastRetryLabel: item?.retryLabel || retryLabel, retryAttempt: attempt, retryMaxAttempts: maxAttempts, retryDelayMs: item?.retryDelayMs ?? 0, lastStatus: item?.status ?? 0, lastStatusText: item?.statusText ?? "request-error", retryable, cookiePresent: false, retryExhausted: false, lastError: item?.error || null, valuesRedacted: true } }).catch(() => {}); - if (!retryable) break; + if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs); } - if (attempt < maxAttempts && attempts[attempts.length - 1]?.retryable === true) await sleep(retryDelayMs); + } finally { + if (closeAuthPage && activeAuthPage && !activeAuthPage.isClosed()) await activeAuthPage.close().catch(() => {}); } const cookieState = await readAuthCookieState(browserContext); const last = attempts[attempts.length - 1] || null; @@ -504,6 +510,25 @@ async function authenticate(browserContext) { throw error; } +async function pageAuthLogin(authPage, loginUrl) { + if (!authPage) throw new Error("auth page is not ready"); + await authPage.goto(loginUrl, { waitUntil: "domcontentloaded", timeout: 12000 }); + return authPage.evaluate(async (input) => { + const response = await fetch("/auth/login", { + method: "POST", + headers: { accept: "application/json", "content-type": "application/json" }, + body: JSON.stringify({ username: input.username, password: input.password }), + credentials: "include", + }); + await response.text().catch(() => ""); + return { + ok: response.ok, + status: response.status, + statusText: response.statusText || "", + }; + }, { username, password }); +} + function publicAuth(value) { if (!value) return null; return { @@ -535,7 +560,7 @@ function isRetryableAuthStatus(status) { function isRetryableAuthError(error) { const message = error && error.message ? String(error.message) : String(error || ""); - return /EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|ECONNABORTED|socket hang up|ERR_NETWORK_CHANGED|fetch failed|network|timeout/iu.test(message); + return /AbortError|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|ECONNABORTED|socket hang up|ERR_NETWORK_CHANGED|fetch failed|network|timeout|aborted/iu.test(message); } function authFailureMessage(failure) { @@ -632,11 +657,11 @@ async function gotoTarget(rawTarget) { const response = await page.goto(target, { waitUntil: "domcontentloaded", timeout: 45000 }); await page.waitForTimeout(1000).catch(() => {}); const httpStatus = response ? response.status() : null; - const readiness = await waitForTargetPageReady(page, target, { timeoutMs: 45000 }); + const readiness = await waitForTargetPageReady(page, target, { timeoutMs: 15000 }); if (!readiness.ok) { - const error = new Error("workbench-app-not-ready: " + (readiness.reason || "target page did not expose the Workbench shell")); - error.navigationReadiness = readiness; - throw error; + const pageProvenance = await refreshPageProvenance("goto-degraded", httpStatus).catch(() => null); + attempts.push({ attempt, ok: false, degraded: true, httpStatus, readiness, failureKind: readiness.reason || "workbench-app-not-ready" }); + return { beforeUrl, afterUrl: currentPageUrl(), httpStatus, pageId, degraded: true, degradedReason: readiness.reason || "workbench-app-not-ready", pageProvenance: compactPageProvenance(pageProvenance), readiness, attempts }; } const pageProvenance = await refreshPageProvenance("goto", httpStatus); attempts.push({ attempt, ok: true, httpStatus, readiness });