diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 9f64a58b..aa6f29a2 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -1557,7 +1557,7 @@ function targetValidationElapsedWarnings(value: unknown, subject: string, budget const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null; const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000; if (elapsedMs === null || elapsedMs <= budgetMs) return []; - return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s targetValidation budget (${Math.round(elapsedMs / 1000)}s); investigate Code Agent multi-round continuity before retrying.`]; + return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s targetValidation budget (${Math.round(elapsedMs / 1000)}s); non-blocking timing alert, only Code Agent multi-round business failures should block acceptance.`]; } function mergeWarnings(...items: readonly (readonly unknown[] | unknown)[]): string[] { @@ -1726,19 +1726,32 @@ function runSentinelValidate(state: SentinelCicdState, options: Extract { let httpStatus = null; let navigationError = null; -try { - const response = await page.goto(url, { timeout, waitUntil: "domcontentloaded" }); - httpStatus = response?.status() ?? null; - await page.waitForLoadState("networkidle", { timeout: Math.min(15000, timeout) }).catch(() => {}); - await page.waitForFunction(() => { - const root = document.querySelector("#sentinel-dashboard"); - if (!root) return false; - const error = document.querySelector("#error-banner"); - const runs = document.querySelectorAll("#runs-body tr").length; - const statusText = document.querySelector("#status-pill")?.textContent || ""; - return (error && !error.hidden) || runs > 0 || (statusText.trim() && statusText.trim() !== "空闲"); - }, null, { timeout: Math.min(15000, timeout) }).catch(() => {}); - await page.waitForTimeout(500); -} catch (error) { - navigationError = String(error?.message || error).slice(0, 500); +let navigationAttempts = 0; +const maxNavigationAttempts = 3; +const perAttemptTimeout = Math.max(5000, Math.floor(timeout / maxNavigationAttempts)); +for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) { + navigationAttempts = attempt; + navigationError = null; + try { + const response = await page.goto(url, { timeout: perAttemptTimeout, waitUntil: "domcontentloaded" }); + httpStatus = response?.status() ?? null; + await page.waitForLoadState("networkidle", { timeout: Math.min(5000, perAttemptTimeout) }).catch(() => {}); + await page.waitForFunction(() => { + const root = document.querySelector("#sentinel-dashboard"); + if (!root) return false; + const error = document.querySelector("#error-banner"); + const runs = document.querySelectorAll("#runs-body tr").length; + const statusText = document.querySelector("#status-pill")?.textContent || ""; + return (error && !error.hidden) || runs > 0 || (statusText.trim() && statusText.trim() !== "空闲"); + }, null, { timeout: Math.min(5000, perAttemptTimeout) }).catch(() => {}); + await page.waitForTimeout(500); + const shellReady = await page.evaluate(() => Boolean(document.querySelector("#sentinel-dashboard"))).catch(() => false); + if (shellReady || attempt === maxNavigationAttempts) break; + } catch (error) { + navigationError = String(error?.message || error).slice(0, 500); + if (attempt === maxNavigationAttempts) break; + } + await page.waitForTimeout(750 * attempt); } if (captureScreenshot && screenshotPath) { @@ -2001,6 +2032,7 @@ console.log("__WEB_PROBE_SENTINEL_DASHBOARD_JSON__" + JSON.stringify({ url, httpStatus, navigationError, + navigationAttempts, executablePath: executablePath || null, viewport: { width, height }, screenshotPath: captureScreenshot ? screenshotPath : null, @@ -2473,7 +2505,7 @@ function recordQuickVerify(state: SentinelCicdState, payload: Record): Record = {}; for (const [key, value] of Object.entries(views)) { const item = record(value); + const limit = key === "summary" || key === "auth-session-switch-summary" ? 8_000 : 6_000; compacted[key] = { ...item, - renderedText: boundQuickVerifyRecordText(item.renderedText, key === "summary" ? 12_000 : 16_000), + renderedText: boundQuickVerifyRecordText(item.renderedText, limit), valuesRedacted: true, }; } return compacted; } +function compactQuickVerifyRecordAnalysis(value: unknown): Record | null { + const item = record(value); + if (Object.keys(item).length === 0) return null; + return { + ok: item.ok === true ? true : item.ok === false ? false : null, + reportOk: item.reportOk === true ? true : item.reportOk === false ? false : null, + reason: stringAtNullable(item, "reason"), + stateDir: stringAtNullable(item, "stateDir"), + reportJsonSha256: stringAtNullable(item, "reportJsonSha256"), + reportMdSha256: stringAtNullable(item, "reportMdSha256"), + findingCount: numberAtNullable(item, "findingCount"), + artifactCount: numberAtNullable(item, "artifactCount"), + counts: compactQuickVerifyRecordCounts(record(item.counts)), + screenshot: compactQuickVerifyRecordScreenshot(record(item.screenshot)), + findings: Array.isArray(item.findings) ? item.findings.slice(0, 16).map(compactQuickVerifyRecordFinding) : [], + pagePerformanceSlowApi: Array.isArray(item.pagePerformanceSlowApi) ? item.pagePerformanceSlowApi.slice(0, 6).map(record) : [], + valuesRedacted: true, + }; +} + +function compactQuickVerifyRecordCounts(value: Record): Record { + return { + samples: numberAtNullable(value, "samples"), + control: numberAtNullable(value, "control"), + network: numberAtNullable(value, "network"), + console: numberAtNullable(value, "console"), + errors: numberAtNullable(value, "errors"), + artifacts: numberAtNullable(value, "artifacts"), + valuesRedacted: true, + }; +} + +function compactQuickVerifyRecordScreenshot(value: Record): Record | null { + if (Object.keys(value).length === 0) return null; + return { + path: stringAtNullable(value, "path"), + sha256: stringAtNullable(value, "sha256"), + bytes: numberAtNullable(value, "bytes"), + valuesRedacted: true, + }; +} + +function compactQuickVerifyRecordFinding(value: unknown): Record { + const item = record(value); + return { + id: stringAtNullable(item, "id"), + kind: stringAtNullable(item, "kind"), + code: stringAtNullable(item, "code"), + severity: stringAtNullable(item, "severity"), + level: stringAtNullable(item, "level"), + count: numberAtNullable(item, "count"), + summary: boundQuickVerifyRecordText(item.summary ?? item.message, 220), + blocking: item.blocking === true, + valuesRedacted: true, + }; +} + function compactQuickVerifyRecordStep(value: unknown): Record { const item = record(value); return { @@ -2533,8 +2623,8 @@ function compactQuickVerifyRecordStepResult(value: Record): Rec timedOut: value.timedOut === true ? true : value.timedOut === false ? false : null, stdoutBytes: numberAtNullable(value, "stdoutBytes"), stderrBytes: numberAtNullable(value, "stderrBytes"), - stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 500), - stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 500), + stdoutPreview: boundQuickVerifyRecordText(value.stdoutPreview, 240), + stderrPreview: boundQuickVerifyRecordText(value.stderrPreview, 240), valuesRedacted: true, }; } @@ -2594,6 +2684,46 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p }; } +function probePublicSentinelService(state: SentinelCicdState, pathWithQuery: string, timeoutSeconds: number): Record { + const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl").replace(/\/$/u, ""); + const url = `${publicBaseUrl}${pathWithQuery.startsWith("/") ? pathWithQuery : `/${pathWithQuery}`}`; + const timeoutMs = Math.max(1000, Math.min(Math.trunc(timeoutSeconds * 1000), 20_000)); + const js = [ + "const url=process.env.REQ_URL||'';", + "const timeoutMs=Number(process.env.REQ_TIMEOUT_MS||10000);", + "let out;", + "try{", + " const controller=new AbortController();", + " const timer=setTimeout(()=>controller.abort(), timeoutMs);", + " const started=Date.now();", + " const res=await fetch(url,{signal:controller.signal});", + " const text=await res.text();", + " clearTimeout(timer);", + " let bodyJson=null; try{bodyJson=JSON.parse(text)}catch{}", + " out={ok:res.ok,httpStatus:res.status,publicUrl:url,contentType:res.headers.get('content-type'),bodyJson,bodyTextPreview:text.slice(0,4000),bodyBytes:Buffer.byteLength(text),elapsedMs:Date.now()-started,valuesRedacted:true};", + "}catch(error){out={ok:false,publicUrl:url,error:error instanceof Error?error.message:String(error),valuesRedacted:true};}", + "console.log(JSON.stringify(out));", + ].join(""); + const result = runCommand(["bun", "-e", js], repoRoot, { + timeoutMs: timeoutMs + 2000, + env: { ...process.env, REQ_URL: url, REQ_TIMEOUT_MS: String(timeoutMs) }, + }); + const parsed = parseJsonObject(result.stdout); + return { + ok: result.exitCode === 0 && parsed?.ok === true, + method: "GET", + path: pathWithQuery, + publicUrl: url, + httpStatus: parsed?.httpStatus ?? null, + bodyJson: record(parsed?.bodyJson), + bodyTextPreview: typeof parsed?.bodyTextPreview === "string" ? parsed.bodyTextPreview : "", + bodyBytes: parsed?.bodyBytes ?? null, + error: parsed?.error ?? null, + result: compactCommand(result), + valuesRedacted: true, + }; +} + function probeSentinelPublicExposure(state: SentinelCicdState, timeoutSeconds: number): Record { const publicBaseUrl = stringAt(state.publicExposure, "publicBaseUrl"); const hostname = stringAt(state.publicExposure, "hostname"); @@ -3749,14 +3879,14 @@ function renderValidateResult(result: Record): string { const quickVerify = record(result.quickVerify); const blocker = record(result.blocker); const next = record(result.next); - const warnings = Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []; + const warnings = mergeWarnings(Array.isArray(result.warnings) ? result.warnings : [], Array.isArray(quickVerify.warnings) ? quickVerify.warnings : []); return [ String(result.command), "", table(["NODE", "LANE", "STATUS", "MODE"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode ?? "status"]]), "", table(["CHECK", "OK", "DETAIL"], [ - ["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl)}`], + ["health", health.ok, `${health.httpStatus ?? "-"} ${short(health.internalUrl ?? health.publicUrl)}`], ["metrics", metrics.ok && metricNames(metrics.bodyTextPreview).includes("web_probe_sentinel_health"), `bytes=${metrics.bodyBytes ?? "-"} metric=web_probe_sentinel_health`], ["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`], ["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`], diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 5e1a3f51..f9278a6c 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -119,8 +119,8 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe } else if (sentinelActionRaw === "dashboard") { const dashboardAction = parseWebProbeSentinelDashboardAction(args[1]); const timeoutMs = positiveIntegerOption(args, "--timeout-ms", 30000, 120000); - const waitTimeoutMs = positiveIntegerOption(args, "--wait-timeout-ms", Math.max(90000, timeoutMs + 30000), 600000); - const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.ceil(waitTimeoutMs / 1000) + 45, 900); + const waitTimeoutMs = positiveIntegerOption(args, "--wait-timeout-ms", Math.max(60000, timeoutMs + 15000), 600000); + const commandTimeoutSeconds = positiveIntegerOption(args, "--command-timeout-seconds", Math.min(900, Math.max(120, Math.ceil(waitTimeoutMs / 1000) + 30)), 900); sentinel = { kind: "dashboard", action: dashboardAction, @@ -134,7 +134,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe waitTimeoutMs, timeoutSeconds: commandTimeoutSeconds, commandTimeoutSeconds, - fullPage: !args.includes("--no-full-page"), + fullPage: args.includes("--full-page") && !args.includes("--no-full-page"), raw: args.includes("--raw"), }; } else { diff --git a/scripts/src/ssh-playwright.ts b/scripts/src/ssh-playwright.ts index 55571bee..acda097c 100644 --- a/scripts/src/ssh-playwright.ts +++ b/scripts/src/ssh-playwright.ts @@ -87,21 +87,33 @@ export async function runSshPlaywrightOperation( for (const artifact of manifest.artifacts) { const localPath = join(localDir, `${runId}-${safePathSegment(basename(artifact.remotePath) || "artifact")}`); - try { - const download = await downloadSshFileVerified( - invocation, - executor, - builders, - artifact.remotePath, - localPath, - options.inactivityTimeoutMs, - ); - artifacts.push({ ...download, manifestBytes: artifact.bytes, manifestSha256: artifact.sha256 }); - } catch (error) { + const maxAttempts = 3; + let downloaded = false; + let lastError: unknown = null; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + try { + const download = await downloadSshFileVerified( + invocation, + executor, + builders, + artifact.remotePath, + localPath, + options.inactivityTimeoutMs, + ); + artifacts.push({ ...download, manifestBytes: artifact.bytes, manifestSha256: artifact.sha256 }); + downloaded = true; + break; + } catch (error) { + lastError = error; + if (attempt < maxAttempts) await sleep(750 * attempt); + } + } + if (!downloaded) { downloadFailure = { remotePath: artifact.remotePath, - message: error instanceof Error ? error.message : String(error), - name: error instanceof Error ? error.name : undefined, + attempts: maxAttempts, + message: lastError instanceof Error ? lastError.message : String(lastError), + name: lastError instanceof Error ? lastError.name : undefined, }; break; } @@ -536,6 +548,10 @@ function decodeBase64(value: string): string { } } +function sleep(ms: number): Promise { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + function safePathSegment(value: string): string { const cleaned = value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, ""); return cleaned.length > 0 ? cleaned.slice(0, 120) : "artifact";