From 56f1ed44a4299b6feacaa4aa59cb3a038dbb1dc0 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 21 Jun 2026 20:46:15 +0000 Subject: [PATCH] fix: route web-probe through YAML proxy --- scripts/src/hwlab-node-impl.ts | 129 +++++++++++++++++- .../hwlab-node-web-observe-runner-source.ts | 60 +++++++- .../src/hwlab-node-web-probe-runner-source.ts | 61 ++++++++- 3 files changed, 244 insertions(+), 6 deletions(-) diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 8d0c9c05..e7b2e12f 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -6727,9 +6727,10 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential); if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential); const probeArgs = nodeWebProbeRunArgs(options, "run"); + const webProbeProxy = nodeWebProbeHostProxyEnv(spec); const script = [ "set -eu", - `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password)} ${probeArgs.map(shellQuote).join(" ")}`, + `${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`, ].join("\n"); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); const probe = compactWebProbeResult(parseJsonObject(result.stdout)); @@ -6748,6 +6749,7 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record lane: options.lane, workspace: spec.workspace, url: options.url, + network: webProbeProxy.summary, credential, commandTimeout: { seconds: options.commandTimeoutSeconds, @@ -6791,9 +6793,10 @@ function runNodeWebProbeAsync( credential: Record, ): Record { const startArgs = nodeWebProbeRunArgs(options, "start"); + const webProbeProxy = nodeWebProbeHostProxyEnv(spec); const startScript = [ "set -eu", - `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password ?? "")} ${startArgs.map(shellQuote).join(" ")}`, + `${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`, ].join("\n"); const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55); const start = parseJsonObject(startResult.stdout); @@ -6807,6 +6810,7 @@ function runNodeWebProbeAsync( lane: options.lane, workspace: spec.workspace, url: options.url, + network: webProbeProxy.summary, credential, mode: "async-start", commandTimeout: webProbeCommandTimeoutSummary(options, false), @@ -6841,6 +6845,7 @@ function runNodeWebProbeAsync( lane: options.lane, workspace: spec.workspace, url: options.url, + network: webProbeProxy.summary, credential, mode: "async", commandTimeout: webProbeCommandTimeoutSummary(options, poll.timedOut), @@ -6947,6 +6952,116 @@ function webProbeCredential(secretSpec: RuntimeSecretSpec, material: BootstrapAd }; } +interface NodeWebProbeHostProxyEnv { + readonly envAssignments: string[]; + readonly summary: Record; +} + +function nodeWebProbeHostProxyEnv(spec: HwlabRuntimeLaneSpec): NodeWebProbeHostProxyEnv { + const proxy = spec.networkProfile.proxy; + const serviceCache = new Map(); + const http = resolveNodeWebProbeHostProxyUrl(spec, proxy.http, serviceCache); + const https = resolveNodeWebProbeHostProxyUrl(spec, proxy.https, serviceCache); + const all = resolveNodeWebProbeHostProxyUrl(spec, proxy.all, serviceCache); + const noProxy = proxy.noProxy.join(","); + return { + envAssignments: [ + ["HTTP_PROXY", http.url], + ["HTTPS_PROXY", https.url], + ["ALL_PROXY", all.url], + ["http_proxy", http.url], + ["https_proxy", https.url], + ["all_proxy", all.url], + ["NO_PROXY", noProxy], + ["no_proxy", noProxy], + ].map(([key, value]) => `${key}=${shellQuote(value)}`), + summary: { + source: "yaml", + mode: "host-env", + networkProfileId: spec.networkProfileId, + proxy: { + http: http.summary, + https: https.summary, + all: all.summary, + noProxyCount: proxy.noProxy.length, + }, + valuesPrinted: false, + }, + }; +} + +function resolveNodeWebProbeHostProxyUrl( + spec: HwlabRuntimeLaneSpec, + rawUrl: string, + serviceCache: Map, +): { url: string; summary: Record } { + let parsed: URL; + try { + parsed = new URL(rawUrl); + } catch (error) { + throw new Error(`config/hwlab-node-lanes.yaml networkProfiles.${spec.networkProfileId}.proxy contains invalid proxy URL: ${error instanceof Error ? error.message : String(error)}`); + } + const service = parseKubernetesServiceDnsHost(parsed.hostname); + if (service === null) { + return { + url: rawUrl, + summary: { + mode: "host-url", + host: parsed.hostname, + port: parsed.port || null, + valuesPrinted: false, + }, + }; + } + const clusterIp = resolveKubernetesServiceClusterIp(spec, service.namespace, service.name, serviceCache); + const originalHost = parsed.hostname; + parsed.hostname = clusterIp; + const resolvedUrl = normalizedProxyUrl(parsed); + return { + url: resolvedUrl, + summary: { + mode: "k8s-service-cluster-ip", + service: service.name, + namespace: service.namespace, + originalHost, + resolvedHost: clusterIp, + port: parsed.port || null, + valuesPrinted: false, + }, + }; +} + +function parseKubernetesServiceDnsHost(hostname: string): { name: string; namespace: string } | null { + const match = hostname.toLowerCase().match(/^([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.svc(?:\.cluster\.local)?$/u); + if (match === null) return null; + return { name: match[1] ?? "", namespace: match[3] ?? "" }; +} + +function resolveKubernetesServiceClusterIp( + spec: HwlabRuntimeLaneSpec, + namespace: string, + serviceName: string, + serviceCache: Map, +): string { + const cacheKey = `${namespace}/${serviceName}`; + const cached = serviceCache.get(cacheKey); + if (cached !== undefined) return cached; + const result = runCommand([transPath(), spec.nodeKubeRoute, "get", "svc", "-n", namespace, serviceName, "-o", "jsonpath={.spec.clusterIP}"], repoRoot, { timeoutMs: 20_000 }); + const clusterIp = result.stdout.trim(); + if (result.exitCode !== 0 || clusterIp.length === 0) { + const reason = result.stderr.trim().slice(-500) || result.stdout.trim().slice(-500) || `exitCode=${result.exitCode}`; + throw new Error(`web-probe proxy service resolution failed for ${spec.nodeId}/${spec.lane} ${namespace}/${serviceName}: ${reason}`); + } + serviceCache.set(cacheKey, clusterIp); + return clusterIp; +} + +function normalizedProxyUrl(parsed: URL): string { + const value = parsed.toString(); + if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, ""); + return value; +} + function runNodeWebProbeObserve( options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, @@ -6977,6 +7092,7 @@ function runNodeWebProbeObserveStart( const day = timestamp.slice(0, 8); const stateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`; const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64"); + const webProbeProxy = nodeWebProbeHostProxyEnv(spec); const script = [ "set -eu", `state_dir=${shellQuote(stateDir)}`, @@ -6986,6 +7102,7 @@ function runNodeWebProbeObserveStart( `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`, "chmod 700 \"$runner\"", [ + ...webProbeProxy.envAssignments, `HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`, @@ -7029,6 +7146,7 @@ function runNodeWebProbeObserveStart( lane: options.lane, workspace: spec.workspace, url: options.url, + network: webProbeProxy.summary, targetPath: options.targetPath, id: observerId, credential, @@ -8172,7 +8290,8 @@ function runNodeWebProbeScript( material: BootstrapAdminPasswordMaterial, credential: Record, ): Record { - const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? ""); + const webProbeProxy = nodeWebProbeHostProxyEnv(spec); + const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? "", webProbeProxy); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); const stdoutReport = parseJsonObject(result.stdout); const runPaths = webProbeScriptRunPathsFromStderr(result.stderr); @@ -8250,6 +8369,7 @@ function runNodeWebProbeScript( lane: options.lane, workspace: spec.workspace, url: options.url, + network: webProbeProxy.summary, credential, scriptSource: options.scriptSource, degradedReason, @@ -8274,7 +8394,7 @@ function runNodeWebProbeScript( }); } -function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string { +function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string, webProbeProxy: NodeWebProbeHostProxyEnv): string { const userScriptB64 = Buffer.from(options.scriptText, "utf8").toString("base64"); const runnerB64 = Buffer.from(nodeWebProbeScriptRunnerSource(), "utf8").toString("base64"); return [ @@ -8291,6 +8411,7 @@ function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secre `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$user_script" ${shellQuote(userScriptB64)}`, `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`, [ + ...webProbeProxy.envAssignments, `HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(password)}`, diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 938332b9..fac1449e 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -21,6 +21,7 @@ const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_ const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000); const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0); const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900"); +const playwrightProxy = proxyConfigFromEnv(baseUrl); const pageId = "page-" + randomBytes(4).toString("hex"); const dirs = { commandsPending: path.join(stateDir, "commands", "pending"), @@ -63,7 +64,7 @@ try { const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href); const { chromium } = await launcher.importPlaywright(); browser = await launcher.launchChromium(chromium); - context = await browser.newContext({ viewport }); + context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) }); auth = await runControlCommand({ id: "startup-login", type: "login", createdAt: startedAt, source: "startup" }, async () => authenticate(context)); page = await context.newPage(); attachPassiveListeners(page); @@ -109,6 +110,7 @@ async function writeManifest(extra = {}) { stateDir, baseUrl, targetPath, + network: publicNetwork(playwrightProxy), pageAuthority: { browser: "chromium", context: "single", pageId, continuityBreaksRecorded: true }, pageProvenance: compactPageProvenance(currentPageProvenance), sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerInitiatedDefault: false, responseBodyReadDefault: false }, @@ -283,6 +285,62 @@ function publicAuth(value) { return { ok: value.ok === true, method: value.method, status: value.status, cookiePresent: value.cookiePresent === true, cookieNames: value.cookieNames || [], valuesRedacted: true }; } +function proxyConfigFromEnv(targetBaseUrl) { + let target; + try { + target = new URL(targetBaseUrl); + } catch { + return null; + } + const noProxy = process.env.NO_PROXY || process.env.no_proxy || ""; + if (noProxyMatches(target.hostname, noProxy)) return null; + const raw = target.protocol === "https:" + ? process.env.HTTPS_PROXY || process.env.https_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTP_PROXY || process.env.http_proxy || "" + : process.env.HTTP_PROXY || process.env.http_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTPS_PROXY || process.env.https_proxy || ""; + if (!raw) return null; + return { server: raw }; +} + +function noProxyMatches(hostname, rawList) { + const host = String(hostname || "").toLowerCase(); + if (!host) return false; + return String(rawList || "").split(",").some((raw) => { + let item = raw.trim().toLowerCase(); + if (!item) return false; + if (item === "*") return true; + item = item.replace(/^\*\./u, "."); + const portIndex = item.lastIndexOf(":"); + if (portIndex > -1 && !item.includes("]")) item = item.slice(0, portIndex); + if (item.startsWith(".")) return host === item.slice(1) || host.endsWith(item); + return host === item; + }); +} + +function publicNetwork(proxy) { + return { + proxy: proxy === null ? { enabled: false, source: "env", valuesRedacted: true } : { + enabled: true, + source: "env", + server: publicProxyServer(proxy.server), + valuesRedacted: true, + }, + valuesRedacted: true, + }; +} + +function publicProxyServer(raw) { + try { + const parsed = new URL(String(raw || "")); + parsed.username = ""; + parsed.password = ""; + const value = parsed.toString(); + if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, ""); + return value; + } catch { + return String(raw || "").replace(/\/\/[^/@]+@/u, "//[redacted]@"); + } +} + async function gotoTarget(rawTarget) { const target = new URL(String(rawTarget || targetPath), baseUrl).toString(); const beforeUrl = currentPageUrl(); diff --git a/scripts/src/hwlab-node-web-probe-runner-source.ts b/scripts/src/hwlab-node-web-probe-runner-source.ts index a53ac4ab..e616b0c5 100644 --- a/scripts/src/hwlab-node-web-probe-runner-source.ts +++ b/scripts/src/hwlab-node-web-probe-runner-source.ts @@ -17,6 +17,7 @@ const runDir = path.resolve(process.env.UNIDESK_WEB_PROBE_RUN_DIR || ".state/web const userScript = path.resolve(process.env.UNIDESK_WEB_PROBE_USER_SCRIPT || path.join(runDir, "user-script.mjs")); const timeoutMs = positiveInteger(process.env.UNIDESK_WEB_PROBE_TIMEOUT_MS, 30000); const viewport = parseViewport(process.env.UNIDESK_WEB_PROBE_VIEWPORT || "1440x900"); +const playwrightProxy = proxyConfigFromEnv(baseUrl); const artifactRecords = []; const readinessRecords = []; const stepRecords = []; @@ -34,7 +35,7 @@ try { const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href); const { chromium } = await launcher.importPlaywright(); browser = await launcher.launchChromium(chromium); - context = await browser.newContext({ viewport }); + context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) }); auth = await authenticate(context); if (!auth.ok) throw new Error("auth-login-failed"); page = await context.newPage(); @@ -55,6 +56,7 @@ try { generatedAt: new Date().toISOString(), startedAt, baseUrl, + network: publicNetwork(playwrightProxy), finalUrl: lastUrl, lastUrl, scriptSha256: userScriptSha256, @@ -83,6 +85,7 @@ try { generatedAt: new Date().toISOString(), startedAt, baseUrl, + network: publicNetwork(playwrightProxy), finalUrl: lastUrl, lastUrl, scriptSha256: userScriptSha256, @@ -365,6 +368,62 @@ function scriptHelpers() { return helpers; } +function proxyConfigFromEnv(targetBaseUrl) { + let target; + try { + target = new URL(targetBaseUrl); + } catch { + return null; + } + const noProxy = process.env.NO_PROXY || process.env.no_proxy || ""; + if (noProxyMatches(target.hostname, noProxy)) return null; + const raw = target.protocol === "https:" + ? process.env.HTTPS_PROXY || process.env.https_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTP_PROXY || process.env.http_proxy || "" + : process.env.HTTP_PROXY || process.env.http_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTPS_PROXY || process.env.https_proxy || ""; + if (!raw) return null; + return { server: raw }; +} + +function noProxyMatches(hostname, rawList) { + const host = String(hostname || "").toLowerCase(); + if (!host) return false; + return String(rawList || "").split(",").some((raw) => { + let item = raw.trim().toLowerCase(); + if (!item) return false; + if (item === "*") return true; + item = item.replace(/^\*\./u, "."); + const portIndex = item.lastIndexOf(":"); + if (portIndex > -1 && !item.includes("]")) item = item.slice(0, portIndex); + if (item.startsWith(".")) return host === item.slice(1) || host.endsWith(item); + return host === item; + }); +} + +function publicNetwork(proxy) { + return { + proxy: proxy === null ? { enabled: false, source: "env", valuesRedacted: true } : { + enabled: true, + source: "env", + server: publicProxyServer(proxy.server), + valuesRedacted: true, + }, + valuesRedacted: true, + }; +} + +function publicProxyServer(raw) { + try { + const parsed = new URL(String(raw || "")); + parsed.username = ""; + parsed.password = ""; + const value = parsed.toString(); + if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, ""); + return value; + } catch { + return String(raw || "").replace(/\/\/[^/@]+@/u, "//[redacted]@"); + } +} + function createLivePageProxy() { return new Proxy({}, { get(_target, prop) {