fix: route web-probe through YAML proxy

scripts/src/hwlab-node-impl.ts

@@ -6727,9 +6727,10 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
   if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential);
   if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential);
   const probeArgs = nodeWebProbeRunArgs(options, "run");
+  const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
   const script = [
     "set -eu",
-    `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password)} ${probeArgs.map(shellQuote).join(" ")}`,
+    `${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password)}`].join(" ")} ${probeArgs.map(shellQuote).join(" ")}`,
   ].join("\n");
   const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
   const probe = compactWebProbeResult(parseJsonObject(result.stdout));
@@ -6748,6 +6749,7 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
     lane: options.lane,
     workspace: spec.workspace,
     url: options.url,
+    network: webProbeProxy.summary,
     credential,
     commandTimeout: {
       seconds: options.commandTimeoutSeconds,
@@ -6791,9 +6793,10 @@ function runNodeWebProbeAsync(
   credential: Record<string, unknown>,
 ): Record<string, unknown> {
   const startArgs = nodeWebProbeRunArgs(options, "start");
+  const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
   const startScript = [
     "set -eu",
-    `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password ?? "")} ${startArgs.map(shellQuote).join(" ")}`,
+    `${[...webProbeProxy.envAssignments, `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`, `HWLAB_WEB_PASS=${shellQuote(material.password ?? "")}`].join(" ")} ${startArgs.map(shellQuote).join(" ")}`,
   ].join("\n");
   const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55);
   const start = parseJsonObject(startResult.stdout);
@@ -6807,6 +6810,7 @@ function runNodeWebProbeAsync(
       lane: options.lane,
       workspace: spec.workspace,
       url: options.url,
+      network: webProbeProxy.summary,
       credential,
       mode: "async-start",
       commandTimeout: webProbeCommandTimeoutSummary(options, false),
@@ -6841,6 +6845,7 @@ function runNodeWebProbeAsync(
     lane: options.lane,
     workspace: spec.workspace,
     url: options.url,
+    network: webProbeProxy.summary,
     credential,
     mode: "async",
     commandTimeout: webProbeCommandTimeoutSummary(options, poll.timedOut),
@@ -6947,6 +6952,116 @@ function webProbeCredential(secretSpec: RuntimeSecretSpec, material: BootstrapAd
   };
 }
 
+interface NodeWebProbeHostProxyEnv {
+  readonly envAssignments: string[];
+  readonly summary: Record<string, unknown>;
+}
+
+function nodeWebProbeHostProxyEnv(spec: HwlabRuntimeLaneSpec): NodeWebProbeHostProxyEnv {
+  const proxy = spec.networkProfile.proxy;
+  const serviceCache = new Map<string, string>();
+  const http = resolveNodeWebProbeHostProxyUrl(spec, proxy.http, serviceCache);
+  const https = resolveNodeWebProbeHostProxyUrl(spec, proxy.https, serviceCache);
+  const all = resolveNodeWebProbeHostProxyUrl(spec, proxy.all, serviceCache);
+  const noProxy = proxy.noProxy.join(",");
+  return {
+    envAssignments: [
+      ["HTTP_PROXY", http.url],
+      ["HTTPS_PROXY", https.url],
+      ["ALL_PROXY", all.url],
+      ["http_proxy", http.url],
+      ["https_proxy", https.url],
+      ["all_proxy", all.url],
+      ["NO_PROXY", noProxy],
+      ["no_proxy", noProxy],
+    ].map(([key, value]) => `${key}=${shellQuote(value)}`),
+    summary: {
+      source: "yaml",
+      mode: "host-env",
+      networkProfileId: spec.networkProfileId,
+      proxy: {
+        http: http.summary,
+        https: https.summary,
+        all: all.summary,
+        noProxyCount: proxy.noProxy.length,
+      },
+      valuesPrinted: false,
+    },
+  };
+}
+
+function resolveNodeWebProbeHostProxyUrl(
+  spec: HwlabRuntimeLaneSpec,
+  rawUrl: string,
+  serviceCache: Map<string, string>,
+): { url: string; summary: Record<string, unknown> } {
+  let parsed: URL;
+  try {
+    parsed = new URL(rawUrl);
+  } catch (error) {
+    throw new Error(`config/hwlab-node-lanes.yaml networkProfiles.${spec.networkProfileId}.proxy contains invalid proxy URL: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  const service = parseKubernetesServiceDnsHost(parsed.hostname);
+  if (service === null) {
+    return {
+      url: rawUrl,
+      summary: {
+        mode: "host-url",
+        host: parsed.hostname,
+        port: parsed.port || null,
+        valuesPrinted: false,
+      },
+    };
+  }
+  const clusterIp = resolveKubernetesServiceClusterIp(spec, service.namespace, service.name, serviceCache);
+  const originalHost = parsed.hostname;
+  parsed.hostname = clusterIp;
+  const resolvedUrl = normalizedProxyUrl(parsed);
+  return {
+    url: resolvedUrl,
+    summary: {
+      mode: "k8s-service-cluster-ip",
+      service: service.name,
+      namespace: service.namespace,
+      originalHost,
+      resolvedHost: clusterIp,
+      port: parsed.port || null,
+      valuesPrinted: false,
+    },
+  };
+}
+
+function parseKubernetesServiceDnsHost(hostname: string): { name: string; namespace: string } | null {
+  const match = hostname.toLowerCase().match(/^([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.([a-z0-9]([-a-z0-9]*[a-z0-9])?)\.svc(?:\.cluster\.local)?$/u);
+  if (match === null) return null;
+  return { name: match[1] ?? "", namespace: match[3] ?? "" };
+}
+
+function resolveKubernetesServiceClusterIp(
+  spec: HwlabRuntimeLaneSpec,
+  namespace: string,
+  serviceName: string,
+  serviceCache: Map<string, string>,
+): string {
+  const cacheKey = `${namespace}/${serviceName}`;
+  const cached = serviceCache.get(cacheKey);
+  if (cached !== undefined) return cached;
+  const result = runCommand([transPath(), spec.nodeKubeRoute, "get", "svc", "-n", namespace, serviceName, "-o", "jsonpath={.spec.clusterIP}"], repoRoot, { timeoutMs: 20_000 });
+  const clusterIp = result.stdout.trim();
+  if (result.exitCode !== 0 || clusterIp.length === 0) {
+    const reason = result.stderr.trim().slice(-500) || result.stdout.trim().slice(-500) || `exitCode=${result.exitCode}`;
+    throw new Error(`web-probe proxy service resolution failed for ${spec.nodeId}/${spec.lane} ${namespace}/${serviceName}: ${reason}`);
+  }
+  serviceCache.set(cacheKey, clusterIp);
+  return clusterIp;
+}
+
+function normalizedProxyUrl(parsed: URL): string {
+  const value = parsed.toString();
+  if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, "");
+  return value;
+}
+
 function runNodeWebProbeObserve(
   options: NodeWebProbeObserveOptions,
   spec: HwlabRuntimeLaneSpec,
@@ -6977,6 +7092,7 @@ function runNodeWebProbeObserveStart(
   const day = timestamp.slice(0, 8);
   const stateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
   const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64");
+  const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
   const script = [
     "set -eu",
     `state_dir=${shellQuote(stateDir)}`,
@@ -6986,6 +7102,7 @@ function runNodeWebProbeObserveStart(
     `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
     "chmod 700 \"$runner\"",
     [
+      ...webProbeProxy.envAssignments,
       `HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
       `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`,
       `HWLAB_WEB_PASS=${shellQuote(material.password)}`,
@@ -7029,6 +7146,7 @@ function runNodeWebProbeObserveStart(
     lane: options.lane,
     workspace: spec.workspace,
     url: options.url,
+    network: webProbeProxy.summary,
     targetPath: options.targetPath,
     id: observerId,
     credential,
@@ -8172,7 +8290,8 @@ function runNodeWebProbeScript(
   material: BootstrapAdminPasswordMaterial,
   credential: Record<string, unknown>,
 ): Record<string, unknown> {
-  const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? "");
+  const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
+  const script = nodeWebProbeScriptRemoteShell(options, secretSpec, material.password ?? "", webProbeProxy);
   const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
   const stdoutReport = parseJsonObject(result.stdout);
   const runPaths = webProbeScriptRunPathsFromStderr(result.stderr);
@@ -8250,6 +8369,7 @@ function runNodeWebProbeScript(
     lane: options.lane,
     workspace: spec.workspace,
     url: options.url,
+    network: webProbeProxy.summary,
     credential,
     scriptSource: options.scriptSource,
     degradedReason,
@@ -8274,7 +8394,7 @@ function runNodeWebProbeScript(
   });
 }
 
-function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string {
+function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string, webProbeProxy: NodeWebProbeHostProxyEnv): string {
   const userScriptB64 = Buffer.from(options.scriptText, "utf8").toString("base64");
   const runnerB64 = Buffer.from(nodeWebProbeScriptRunnerSource(), "utf8").toString("base64");
   return [
@@ -8291,6 +8411,7 @@ function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secre
     `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$user_script" ${shellQuote(userScriptB64)}`,
     `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$runner" ${shellQuote(runnerB64)}`,
     [
+      ...webProbeProxy.envAssignments,
       `HWLAB_WEB_BASE_URL=${shellQuote(options.url)}`,
       `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)}`,
       `HWLAB_WEB_PASS=${shellQuote(password)}`,

scripts/src/hwlab-node-web-observe-runner-source.ts

@@ -21,6 +21,7 @@ const sampleIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SAMPLE_
 const screenshotIntervalMs = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_SCREENSHOT_INTERVAL_MS, 300000);
 const maxSamples = positiveInteger(process.env.UNIDESK_WEB_OBSERVE_MAX_SAMPLES, 0);
 const viewport = parseViewport(process.env.UNIDESK_WEB_OBSERVE_VIEWPORT || "1440x900");
+const playwrightProxy = proxyConfigFromEnv(baseUrl);
 const pageId = "page-" + randomBytes(4).toString("hex");
 const dirs = {
   commandsPending: path.join(stateDir, "commands", "pending"),
@@ -63,7 +64,7 @@ try {
   const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
   const { chromium } = await launcher.importPlaywright();
   browser = await launcher.launchChromium(chromium);
-  context = await browser.newContext({ viewport });
+  context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) });
   auth = await runControlCommand({ id: "startup-login", type: "login", createdAt: startedAt, source: "startup" }, async () => authenticate(context));
   page = await context.newPage();
   attachPassiveListeners(page);
@@ -109,6 +110,7 @@ async function writeManifest(extra = {}) {
     stateDir,
     baseUrl,
     targetPath,
+    network: publicNetwork(playwrightProxy),
     pageAuthority: { browser: "chromium", context: "single", pageId, continuityBreaksRecorded: true },
     pageProvenance: compactPageProvenance(currentPageProvenance),
     sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerInitiatedDefault: false, responseBodyReadDefault: false },
@@ -283,6 +285,62 @@ function publicAuth(value) {
   return { ok: value.ok === true, method: value.method, status: value.status, cookiePresent: value.cookiePresent === true, cookieNames: value.cookieNames || [], valuesRedacted: true };
 }
 
+function proxyConfigFromEnv(targetBaseUrl) {
+  let target;
+  try {
+    target = new URL(targetBaseUrl);
+  } catch {
+    return null;
+  }
+  const noProxy = process.env.NO_PROXY || process.env.no_proxy || "";
+  if (noProxyMatches(target.hostname, noProxy)) return null;
+  const raw = target.protocol === "https:"
+    ? process.env.HTTPS_PROXY || process.env.https_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTP_PROXY || process.env.http_proxy || ""
+    : process.env.HTTP_PROXY || process.env.http_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTPS_PROXY || process.env.https_proxy || "";
+  if (!raw) return null;
+  return { server: raw };
+}
+
+function noProxyMatches(hostname, rawList) {
+  const host = String(hostname || "").toLowerCase();
+  if (!host) return false;
+  return String(rawList || "").split(",").some((raw) => {
+    let item = raw.trim().toLowerCase();
+    if (!item) return false;
+    if (item === "*") return true;
+    item = item.replace(/^\*\./u, ".");
+    const portIndex = item.lastIndexOf(":");
+    if (portIndex > -1 && !item.includes("]")) item = item.slice(0, portIndex);
+    if (item.startsWith(".")) return host === item.slice(1) || host.endsWith(item);
+    return host === item;
+  });
+}
+
+function publicNetwork(proxy) {
+  return {
+    proxy: proxy === null ? { enabled: false, source: "env", valuesRedacted: true } : {
+      enabled: true,
+      source: "env",
+      server: publicProxyServer(proxy.server),
+      valuesRedacted: true,
+    },
+    valuesRedacted: true,
+  };
+}
+
+function publicProxyServer(raw) {
+  try {
+    const parsed = new URL(String(raw || ""));
+    parsed.username = "";
+    parsed.password = "";
+    const value = parsed.toString();
+    if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, "");
+    return value;
+  } catch {
+    return String(raw || "").replace(/\/\/[^/@]+@/u, "//[redacted]@");
+  }
+}
+
 async function gotoTarget(rawTarget) {
   const target = new URL(String(rawTarget || targetPath), baseUrl).toString();
   const beforeUrl = currentPageUrl();

scripts/src/hwlab-node-web-probe-runner-source.ts

@@ -17,6 +17,7 @@ const runDir = path.resolve(process.env.UNIDESK_WEB_PROBE_RUN_DIR || ".state/web
 const userScript = path.resolve(process.env.UNIDESK_WEB_PROBE_USER_SCRIPT || path.join(runDir, "user-script.mjs"));
 const timeoutMs = positiveInteger(process.env.UNIDESK_WEB_PROBE_TIMEOUT_MS, 30000);
 const viewport = parseViewport(process.env.UNIDESK_WEB_PROBE_VIEWPORT || "1440x900");
+const playwrightProxy = proxyConfigFromEnv(baseUrl);
 const artifactRecords = [];
 const readinessRecords = [];
 const stepRecords = [];
@@ -34,7 +35,7 @@ try {
   const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
   const { chromium } = await launcher.importPlaywright();
   browser = await launcher.launchChromium(chromium);
-  context = await browser.newContext({ viewport });
+  context = await browser.newContext({ viewport, ...(playwrightProxy === null ? {} : { proxy: playwrightProxy }) });
   auth = await authenticate(context);
   if (!auth.ok) throw new Error("auth-login-failed");
   page = await context.newPage();
@@ -55,6 +56,7 @@ try {
     generatedAt: new Date().toISOString(),
     startedAt,
     baseUrl,
+    network: publicNetwork(playwrightProxy),
     finalUrl: lastUrl,
     lastUrl,
     scriptSha256: userScriptSha256,
@@ -83,6 +85,7 @@ try {
     generatedAt: new Date().toISOString(),
     startedAt,
     baseUrl,
+    network: publicNetwork(playwrightProxy),
     finalUrl: lastUrl,
     lastUrl,
     scriptSha256: userScriptSha256,
@@ -365,6 +368,62 @@ function scriptHelpers() {
   return helpers;
 }
 
+function proxyConfigFromEnv(targetBaseUrl) {
+  let target;
+  try {
+    target = new URL(targetBaseUrl);
+  } catch {
+    return null;
+  }
+  const noProxy = process.env.NO_PROXY || process.env.no_proxy || "";
+  if (noProxyMatches(target.hostname, noProxy)) return null;
+  const raw = target.protocol === "https:"
+    ? process.env.HTTPS_PROXY || process.env.https_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTP_PROXY || process.env.http_proxy || ""
+    : process.env.HTTP_PROXY || process.env.http_proxy || process.env.ALL_PROXY || process.env.all_proxy || process.env.HTTPS_PROXY || process.env.https_proxy || "";
+  if (!raw) return null;
+  return { server: raw };
+}
+
+function noProxyMatches(hostname, rawList) {
+  const host = String(hostname || "").toLowerCase();
+  if (!host) return false;
+  return String(rawList || "").split(",").some((raw) => {
+    let item = raw.trim().toLowerCase();
+    if (!item) return false;
+    if (item === "*") return true;
+    item = item.replace(/^\*\./u, ".");
+    const portIndex = item.lastIndexOf(":");
+    if (portIndex > -1 && !item.includes("]")) item = item.slice(0, portIndex);
+    if (item.startsWith(".")) return host === item.slice(1) || host.endsWith(item);
+    return host === item;
+  });
+}
+
+function publicNetwork(proxy) {
+  return {
+    proxy: proxy === null ? { enabled: false, source: "env", valuesRedacted: true } : {
+      enabled: true,
+      source: "env",
+      server: publicProxyServer(proxy.server),
+      valuesRedacted: true,
+    },
+    valuesRedacted: true,
+  };
+}
+
+function publicProxyServer(raw) {
+  try {
+    const parsed = new URL(String(raw || ""));
+    parsed.username = "";
+    parsed.password = "";
+    const value = parsed.toString();
+    if (parsed.pathname === "/" && parsed.search === "" && parsed.hash === "") return value.replace(/\/$/u, "");
+    return value;
+  } catch {
+    return String(raw || "").replace(/\/\/[^/@]+@/u, "//[redacted]@");
+  }
+}
+
 function createLivePageProxy() {
   return new Proxy({}, {
     get(_target, prop) {