mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: route web-probe through YAML proxy

Browse Source
This commit is contained in:
Codex
2026-06-21 20:46:15 +00:00
parent 625e3a1ab2
commit 56f1ed44a4
3 changed files with 244 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-impl.ts
+125 -4
View File
@@ -6727,9 +6727,10 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential);
if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential);
const probeArgs = nodeWebProbeRunArgs(options, "run");
const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
const script = [
"set -eu",
`HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password)} ${probeArgs.map(shellQuote).join(" ")}`,
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`,
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const probe = compactWebProbeResult(parseJsonObject(result.stdout));
@@ -6748,6 +6749,7 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
lane: options.lane,
workspace: spec.workspace,
url: options.url,
network: webProbeProxy.summary,
credential,
commandTimeout: {
seconds: options.commandTimeoutSeconds,
@@ -6791,9 +6793,10 @@ function runNodeWebProbeAsync(
credential: Record<string, unknown>,
): Record<string, unknown> {
const startArgs = nodeWebProbeRunArgs(options, "start");
const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
const startScript = [
"set -eu",
`HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password ?? "")} ${startArgs.map(shellQuote).join(" ")}`,
`${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`,
].join("\n");
const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55);
const start = parseJsonObject(startResult.stdout);
@@ -6807,6 +6810,7 @@ function runNodeWebProbeAsync(
lane: options.lane,
workspace: spec.workspace,
url: options.url,
network: webProbeProxy.summary,
credential,
mode: "async-start",
commandTimeout: webProbeCommandTimeoutSummary(options, false),
@@ -6841,6 +6845,7 @@ function runNodeWebProbeAsync(
lane: options.lane,
workspace: spec.workspace,
url: options.url,
network: webProbeProxy.summary,
credential,
mode: "async",
commandTimeout: webProbeCommandTimeoutSummary(options, poll.timedOut),
@@ -6947,6 +6952,116 @@ function webProbeCredential(secretSpec: RuntimeSecretSpec, material: BootstrapAd
};
}
interface NodeWebProbeHostProxyEnv {
readonly envAssignments: string[];
readonly summary: Record<string, unknown>;
}
function nodeWebProbeHostProxyEnv(spec: HwlabRuntimeLaneSpec): NodeWebProbeHostProxyEnv {
const proxy = spec.networkProfile.proxy;
const serviceCache = new Map<string, string>();
const http = resolveNodeWebProbeHostProxyUrl(spec, proxy.http, serviceCache);
const https = resolveNodeWebProbeHostProxyUrl(spec, proxy.https, serviceCache);
const all = resolveNodeWebProbeHostProxyUrl(spec, proxy.all, serviceCache);
const noProxy = proxy.noProxy.join(",");
return {
envAssignments: [
["HTTP_PROXY", http.url],
["HTTPS_PROXY", https.url],
["ALL_PROXY", all.url],
["http_proxy", http.url],
["https_proxy", https.url],
["all_proxy", all.url],
["NO_PROXY", noProxy],
["no_proxy", noProxy],
].map(([key, value]) => `${key}=${shellQuote(value)}`),
summary: {
source: "yaml",
mode: "host-env",
networkProfileId: spec.networkProfileId,
proxy: {
http: http.summary,
https: https.summary,
all: all.summary,
noProxyCount: proxy.noProxy.length,
},
valuesPrinted: false,
},
};
}
function resolveNodeWebProbeHostProxyUrl(
spec: HwlabRuntimeLaneSpec,
rawUrl: string,
serviceCache: Map<string, string>,
): { url: string; summary: Record<string, unknown> } {
let parsed: URL;
try {
parsed = new URL(rawUrl);
} catch (error) {
throw new Error(`config/hwlab-node-lanes.yaml networkProfiles.${spec.networkProfileId}.proxy contains invalid proxy URL: ${error instanceof Error ? error.message : String(error)}`);
}
const service = parseKubernetesServiceDnsHost(parsed.hostname);
if (service === null) {
return {
url: rawUrl,
summary: {
mode: "host-url",
host: parsed.hostname,
port: parsed.port || null,
valuesPrinted: false,
},
};
}
const clusterIp = resolveKubernetesServiceClusterIp(spec, service.namespace, service.name, serviceCache);
const originalHost = parsed.hostname;
parsed.hostname = clusterIp;
const resolvedUrl = normalizedProxyUrl(parsed);
return {
url: resolvedUrl,
summary: {
mode: "k8s-service-cluster-ip",
service: service.name,
namespace: service.namespace,
originalHost,
resolvedHost: clusterIp,
port: parsed.port || null,
valuesPrinted: false,
},
};
}
function parseKubernetesServiceDnsHost(hostname: string): { name: string; namespace: string } | null {
const match = hostname.toLowerCase().match(/^([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.svc(?:\.cluster\.local)?$/u);
if (match === null) return null;
return { name: match[1] ?? "", namespace: match[3] ?? "" };
}
function resolveKubernetesServiceClusterIp(
spec: HwlabRuntimeLaneSpec,
namespace: string,
serviceName: string,
serviceCache: Map<string, string>,
): string {
const cacheKey = `${namespace}/${serviceName}`;
const cached = serviceCache.get(cacheKey);
if (cached !== undefined) return cached;
const result = runCommand([transPath(), spec.nodeKubeRoute, "get", "svc", "-n", namespace, serviceName, "-o", "jsonpath={.spec.clusterIP}"], repoRoot, { timeoutMs: 20_000 });
const clusterIp = result.stdout.trim();
if (result.exitCode !== 0 || clusterIp.length === 0) {
const reason = result.stderr.trim().slice(-500) || result.stdout.trim().slice(-500) || `exitCode=${result.exitCode}`;
throw new Error(`web-probe proxy service resolution failed for ${spec.nodeId}/${spec.lane} ${namespace}/${serviceName}: ${reason}`);
}
serviceCache.set(cacheKey, clusterIp);
return clusterIp;
}
function normalizedProxyUrl(parsed: URL): string {
const value = parsed.toString();
if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, "");
return value;
}
function runNodeWebProbeObserve(
options: NodeWebProbeObserveOptions,
spec: HwlabRuntimeLaneSpec,
@@ -6977,6 +7092,7 @@ function runNodeWebProbeObserveStart(
const day = timestamp.slice(0, 8);
const stateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64");
const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
const script = [
"set -eu",
`state_dir=${shellQuote(stateDir)}`,
@@ -6986,6 +7102,7 @@ function runNodeWebProbeObserveStart(
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
"chmod 700 \"$runner\"",
[
...webProbeProxy.envAssignments,
`HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
`HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`,
`HWLAB_WEB_PASS=${shellQuote(material.password)}`,
@@ -7029,6 +7146,7 @@ function runNodeWebProbeObserveStart(
lane: options.lane,
workspace: spec.workspace,
url: options.url,
network: webProbeProxy.summary,
targetPath: options.targetPath,
id: observerId,
credential,
@@ -8172,7 +8290,8 @@ function runNodeWebProbeScript(
material: BootstrapAdminPasswordMaterial,
credential: Record<string, unknown>,
): Record<string, unknown> {
const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? "");
const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? "", webProbeProxy);
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const stdoutReport = parseJsonObject(result.stdout);
const runPaths = webProbeScriptRunPathsFromStderr(result.stderr);
@@ -8250,6 +8369,7 @@ function runNodeWebProbeScript(
lane: options.lane,
workspace: spec.workspace,
url: options.url,
network: webProbeProxy.summary,
credential,
scriptSource: options.scriptSource,
degradedReason,
@@ -8274,7 +8394,7 @@ function runNodeWebProbeScript(
});
}
function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string {
function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string, webProbeProxy: NodeWebProbeHostProxyEnv): string {
const userScriptB64 = Buffer.from(options.scriptText, "utf8").toString("base64");
const runnerB64 = Buffer.from(nodeWebProbeScriptRunnerSource(), "utf8").toString("base64");
return [
@@ -8291,6 +8411,7 @@ function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secre
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$user_script" ${shellQuote(userScriptB64)}`,
`node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
[
...webProbeProxy.envAssignments,
`HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
`HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`,
`HWLAB_WEB_PASS=${shellQuote(password)}`,
scripts/src/hwlab-node-web-observe-runner-source.ts
+59 -1
View File
@@ -21,6 +21,7 @@ const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_
const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000);
const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0);
const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900");
const playwrightProxy = proxyConfigFromEnv(baseUrl);
const pageId = "page-" + randomBytes(4).toString("hex");
const dirs = {
commandsPending: path.join(stateDir, "commands", "pending"),
@@ -63,7 +64,7 @@ try {
const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
const { chromium } = await launcher.importPlaywright();
browser = await launcher.launchChromium(chromium);
context = await browser.newContext({ viewport });
context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) });
auth = await runControlCommand({ id: "startup-login", type: "login", createdAt: startedAt, source: "startup" }, async () => authenticate(context));
page = await context.newPage();
attachPassiveListeners(page);
@@ -109,6 +110,7 @@ async function writeManifest(extra = {}) {
stateDir,
baseUrl,
targetPath,
network: publicNetwork(playwrightProxy),
pageAuthority: { browser: "chromium", context: "single", pageId, continuityBreaksRecorded: true },
pageProvenance: compactPageProvenance(currentPageProvenance),
sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerInitiatedDefault: false, responseBodyReadDefault: false },
@@ -283,6 +285,62 @@ function publicAuth(value) {
return { ok: value.ok === true, method: value.method, status: value.status, cookiePresent: value.cookiePresent === true, cookieNames: value.cookieNames || [], valuesRedacted: true };
}
function proxyConfigFromEnv(targetBaseUrl) {
let target;
try {
target = new URL(targetBaseUrl);
} catch {
return null;
}
const noProxy = process.env.NO_PROXY || process.env.no_proxy || "";
if (noProxyMatches(target.hostname, noProxy)) return null;
const raw = target.protocol === "https:"
? process.env.HTTPS_PROXY || process.env.https_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTP_PROXY || process.env.http_proxy || ""
: process.env.HTTP_PROXY || process.env.http_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTPS_PROXY || process.env.https_proxy || "";
if (!raw) return null;
return { server: raw };
}
function noProxyMatches(hostname, rawList) {
const host = String(hostname || "").toLowerCase();
if (!host) return false;
return String(rawList || "").split(",").some((raw) => {
let item = raw.trim().toLowerCase();
if (!item) return false;
if (item === "*") return true;
item = item.replace(/^\*\./u, ".");
const portIndex = item.lastIndexOf(":");
if (portIndex > -1 && !item.includes("]")) item = item.slice(0, portIndex);
if (item.startsWith(".")) return host === item.slice(1) || host.endsWith(item);
return host === item;
});
}
function publicNetwork(proxy) {
return {
proxy: proxy === null ? { enabled: false, source: "env", valuesRedacted: true } : {
enabled: true,
source: "env",
server: publicProxyServer(proxy.server),
valuesRedacted: true,
},
valuesRedacted: true,
};
}
function publicProxyServer(raw) {
try {
const parsed = new URL(String(raw || ""));
parsed.username = "";
parsed.password = "";
const value = parsed.toString();
if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, "");
return value;
} catch {
return String(raw || "").replace(/\/\/[^/@]+@/u, "//[redacted]@");
}
}
async function gotoTarget(rawTarget) {
const target = new URL(String(rawTarget || targetPath), baseUrl).toString();
const beforeUrl = currentPageUrl();
scripts/src/hwlab-node-web-probe-runner-source.ts
+60 -1
View File
@@ -17,6 +17,7 @@ const runDir = path.resolve(process.env.UNIDESK_WEB_PROBE_RUN_DIR || ".state/web
const userScript = path.resolve(process.env.UNIDESK_WEB_PROBE_USER_SCRIPT || path.join(runDir, "user-script.mjs"));
const timeoutMs = positiveInteger(process.env.UNIDESK_WEB_PROBE_TIMEOUT_MS, 30000);
const viewport = parseViewport(process.env.UNIDESK_WEB_PROBE_VIEWPORT || "1440x900");
const playwrightProxy = proxyConfigFromEnv(baseUrl);
const artifactRecords = [];
const readinessRecords = [];
const stepRecords = [];
@@ -34,7 +35,7 @@ try {
const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
const { chromium } = await launcher.importPlaywright();
browser = await launcher.launchChromium(chromium);
context = await browser.newContext({ viewport });
context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) });
auth = await authenticate(context);
if (!auth.ok) throw new Error("auth-login-failed");
page = await context.newPage();
@@ -55,6 +56,7 @@ try {
generatedAt: new Date().toISOString(),
startedAt,
baseUrl,
network: publicNetwork(playwrightProxy),
finalUrl: lastUrl,
lastUrl,
scriptSha256: userScriptSha256,
@@ -83,6 +85,7 @@ try {
generatedAt: new Date().toISOString(),
startedAt,
baseUrl,
network: publicNetwork(playwrightProxy),
finalUrl: lastUrl,
lastUrl,
scriptSha256: userScriptSha256,
@@ -365,6 +368,62 @@ function scriptHelpers() {
return helpers;
}
function proxyConfigFromEnv(targetBaseUrl) {
let target;
try {
target = new URL(targetBaseUrl);
} catch {
return null;
}
const noProxy = process.env.NO_PROXY || process.env.no_proxy || "";
if (noProxyMatches(target.hostname, noProxy)) return null;
const raw = target.protocol === "https:"
? process.env.HTTPS_PROXY || process.env.https_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTP_PROXY || process.env.http_proxy || ""
: process.env.HTTP_PROXY || process.env.http_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTPS_PROXY || process.env.https_proxy || "";
if (!raw) return null;
return { server: raw };
}
function noProxyMatches(hostname, rawList) {
const host = String(hostname || "").toLowerCase();
if (!host) return false;
return String(rawList || "").split(",").some((raw) => {
let item = raw.trim().toLowerCase();
if (!item) return false;
if (item === "*") return true;
item = item.replace(/^\*\./u, ".");
const portIndex = item.lastIndexOf(":");
if (portIndex > -1 && !item.includes("]")) item = item.slice(0, portIndex);
if (item.startsWith(".")) return host === item.slice(1) || host.endsWith(item);
return host === item;
});
}
function publicNetwork(proxy) {
return {
proxy: proxy === null ? { enabled: false, source: "env", valuesRedacted: true } : {
enabled: true,
source: "env",
server: publicProxyServer(proxy.server),
valuesRedacted: true,
},
valuesRedacted: true,
};
}
function publicProxyServer(raw) {
try {
const parsed = new URL(String(raw || ""));
parsed.username = "";
parsed.password = "";
const value = parsed.toString();
if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, "");
return value;
} catch {
return String(raw || "").replace(/\/\/[^/@]+@/u, "//[redacted]@");
}
}
function createLivePageProxy() {
return new Proxy({}, {
get(_target, prop) {