From fc7047d46774e8aae51fad3c71359ef0e8572b5f Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Wed, 24 Jun 2026 09:40:30 +0800 Subject: [PATCH] fix(hwlab): allow direct ssh git mirror egress (#807) Co-authored-by: Codex --- config/hwlab-node-control-plane.yaml | 4 +- scripts/src/hwlab-node-control-plane.ts | 111 +++++++++++++----------- scripts/src/hwlab-node-impl.ts | 45 ++++++---- 3 files changed, 90 insertions(+), 70 deletions(-) diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 01ea5dfb..135a7f6d 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -114,8 +114,8 @@ targets: readUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git egressProxy: - mode: node-global - required: true + mode: direct + required: false githubTransport: mode: ssh tekton: diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 4b16559b..cd9218d9 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -52,7 +52,7 @@ interface ControlPlaneEgressProxySpec { } interface ControlPlaneGitMirrorEgressProxySpec { - mode: "node-global"; + mode: "node-global" | "direct"; required: boolean; } @@ -859,10 +859,10 @@ function egressProxySpec(raw: Record, path: string): ControlPla function gitMirrorEgressProxySpec(raw: Record, path: string): ControlPlaneGitMirrorEgressProxySpec { const mode = stringField(raw, "mode", path); - if (mode !== "node-global") throw new Error(`${path}.mode must be node-global`); + if (mode !== "node-global" && mode !== "direct") throw new Error(`${path}.mode must be node-global or direct`); return { mode, - required: raw.required === undefined ? true : booleanField(raw, "required", path), + required: raw.required === undefined ? mode !== "direct" : booleanField(raw, "required", path), }; } @@ -1284,58 +1284,37 @@ NODE function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { const gitMirrorProxy = target.gitMirror.egressProxy; const transport = target.gitMirror.githubTransport; - if (gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub transport no longer falls back to localhost`); - if (gitMirrorProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`); - if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub transport no longer falls back to localhost`); - const proxyHost = `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`; - const proxyPort = node.egressProxy.port; - const proxyUrl = `http://${proxyHost}:${proxyPort}`; - const noProxy = node.egressProxy.noProxy.join(","); - const proxySummary = transport.mode === "https" - ? `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=https proxy=HTTP_PROXY authSecret=${transport.tokenSecretName} authKey=${transport.tokenSecretKey} authSourceRef=${transport.tokenSourceRef} source=yaml` - : `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`; - const proxyCommand = `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`; + const useDirect = gitMirrorProxy?.mode === "direct"; + const proxyRequired = gitMirrorProxy?.required === true; + if (!useDirect && gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global or direct`); + if (!useDirect && proxyRequired && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`); + if (!useDirect && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub transport cannot use node-global proxy`); + const proxyHost = useDirect || node.egressProxy === null ? "" : `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`; + const proxyPort = useDirect || node.egressProxy === null ? 0 : node.egressProxy.port; + const proxyUrl = useDirect ? "" : `http://${proxyHost}:${proxyPort}`; + const noProxy = useDirect || node.egressProxy === null ? "" : node.egressProxy.noProxy.join(","); + const proxySummary = useDirect + ? `git-mirror-egress-proxy mode=direct required=false transport=${transport.mode} source=yaml` + : transport.mode === "https" + ? `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=node-global required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=https proxy=HTTP_PROXY authSecret=${transport.tokenSecretName} authKey=${transport.tokenSecretKey} authSourceRef=${transport.tokenSourceRef} source=yaml` + : `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=node-global required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`; + const proxyCommand = useDirect ? "" : `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`; const common = [ `printf '%s\\n' ${shQuote(proxySummary)} >&2`, - `export HTTP_PROXY=${shQuote(proxyUrl)}`, - `export HTTPS_PROXY=${shQuote(proxyUrl)}`, - `export ALL_PROXY=${shQuote(proxyUrl)}`, - `export http_proxy=${shQuote(proxyUrl)}`, - `export https_proxy=${shQuote(proxyUrl)}`, - `export all_proxy=${shQuote(proxyUrl)}`, - `export NO_PROXY=${shQuote(noProxy)}`, - `export no_proxy=${shQuote(noProxy)}`, + useDirect ? "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy" : `export HTTP_PROXY=${shQuote(proxyUrl)}`, + useDirect ? "export NO_PROXY='*'" : `export HTTPS_PROXY=${shQuote(proxyUrl)}`, + useDirect ? "export no_proxy='*'" : `export ALL_PROXY=${shQuote(proxyUrl)}`, + useDirect ? "" : `export http_proxy=${shQuote(proxyUrl)}`, + useDirect ? "" : `export https_proxy=${shQuote(proxyUrl)}`, + useDirect ? "" : `export all_proxy=${shQuote(proxyUrl)}`, + useDirect ? "" : `export NO_PROXY=${shQuote(noProxy)}`, + useDirect ? "" : `export no_proxy=${shQuote(noProxy)}`, `repository=${shQuote(target.source.repository)}`, `source_branch=${shQuote(target.source.branch)}`, `gitops_branch=${shQuote(target.gitops.branch)}`, "repo=\"/cache/${repository}.git\"", - ]; - if (transport.mode === "https") { - return [ - ...common, - "if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo 'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env' >&2; exit 64; fi", - "cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'", - "#!/bin/sh", - "case \"$1\" in", - " *Username*) printf '%s\\n' \"${GITHUB_USERNAME:-x-access-token}\" ;;", - " *Password*) printf '%s\\n' \"$GITHUB_TOKEN\" ;;", - " *) printf '\\n' ;;", - "esac", - "SH_ASKPASS", - "chmod 0700 /tmp/hwlab-git-askpass.sh", - `export GITHUB_USERNAME=${shQuote(transport.username)}`, - "export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh", - "export GIT_TERMINAL_PROMPT=0", - "unset GIT_SSH", - "unset GIT_SSH_COMMAND", - "remote=\"https://github.com/${repository}.git\"", - ].join("\n"); - } - return [ - "mkdir -p /root/.ssh", - "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", - "chmod 0400 /root/.ssh/id_rsa", - ...common, + ].filter(Boolean); + const proxyConnectBlock = useDirect ? [] : [ "cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'", "#!/usr/bin/env node", "const net = require('node:net');", @@ -1387,15 +1366,45 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT "socket.on('data', onData);", "NODE_PROXY", "chmod 0700 /tmp/hwlab-github-proxy-connect.cjs", + ]; + if (transport.mode === "https") { + return [ + ...common, + "if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo 'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env' >&2; exit 64; fi", + "cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'", + "#!/bin/sh", + "case \"$1\" in", + " *Username*) printf '%s\\n' \"${GITHUB_USERNAME:-x-access-token}\" ;;", + " *Password*) printf '%s\\n' \"$GITHUB_TOKEN\" ;;", + " *) printf '\\n' ;;", + "esac", + "SH_ASKPASS", + "chmod 0700 /tmp/hwlab-git-askpass.sh", + `export GITHUB_USERNAME=${shQuote(transport.username)}`, + "export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh", + "export GIT_TERMINAL_PROMPT=0", + "unset GIT_SSH", + "unset GIT_SSH_COMMAND", + "remote=\"https://github.com/${repository}.git\"", + ].join("\n"); + } + return [ + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + ...common, + ...proxyConnectBlock, "cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'", "#!/bin/sh", - `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`, + useDirect + ? `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$@"` + : `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`, "SH_PROXY", "chmod 0700 /tmp/hwlab-git-ssh-proxy.sh", "export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh", "unset GIT_SSH_COMMAND", "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", - ].join("\n"); + ].filter(Boolean).join("\n"); } function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 29ccd5ec..c1900d14 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -274,17 +274,19 @@ type NodeRuntimeGitMirrorGithubTransportSpec = tokenSourceKey: string; }; -interface NodeRuntimeGitMirrorEgressProxySpec { - mode: "k8s-service-cluster-ip"; - clientName: string; - namespace: string; - serviceName: string; - port: number; - sourceRef: string; - sourceKey: string; - sourceType: "subscription-url"; - noProxy: string[]; -} +type NodeRuntimeGitMirrorEgressProxySpec = + | { mode: "direct"; required: false } + | { + mode: "k8s-service-cluster-ip"; + clientName: string; + namespace: string; + serviceName: string; + port: number; + sourceRef: string; + sourceKey: string; + sourceType: "subscription-url"; + noProxy: string[]; + }; const MASTER_ADMIN_API_KEY_KEY = "api-key"; const BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY = "password-hash"; @@ -3386,6 +3388,7 @@ function nodeRuntimeGitMirrorRetryableFailure( ): Record | null { const text = `${result.stdout}\n${result.stderr}`; const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text); + const sshProxyBannerFailure = mirror.egressProxy.mode !== "direct" && /Connection timed out during banner exchange|Connection to UNKNOWN port 65535 timed out|kex_exchange_identification|Connection closed by remote host/iu.test(text); const waitTimeoutFailure = result.exitCode === 45 || result.exitCode === 124 || /UNIDESK_SSH_RUNTIME_TIMEOUT|ssh-runtime-timeout|ssh\/tran operation exceeded|job,pod/iu.test(text); @@ -3437,6 +3440,7 @@ function nodeRuntimeGitMirrorRetryableFailure( } const retryable = partialSuccess !== null || proxyConnectFailure + || sshProxyBannerFailure || waitTimeoutFailure || /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text); if (!retryable) return null; @@ -3455,8 +3459,8 @@ function nodeRuntimeGitMirrorRetryableFailure( ? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback." : waitTimeoutFailure ? "Git mirror job wait exceeded the controlled short-connection budget. Standard git-mirror retries with exponential backoff and stops when retry budget is exhausted." - : proxyConnectFailure - ? "Git mirror job hit a retryable YAML-first proxy CONNECT failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted." + : proxyConnectFailure || sshProxyBannerFailure + ? "Git mirror job hit a retryable YAML-first SSH-over-proxy failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted." : `Git mirror job hit a retryable upstream GitHub ${mirror.githubTransport.mode} transport/fetch failure. Standard git-mirror stops without host workspace fallback.`, refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror), @@ -3860,6 +3864,7 @@ function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record[] { const proxy = mirror.egressProxy; + if (proxy.mode === "direct") return []; const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`; const noProxy = proxy.noProxy.join(","); return [ @@ -5930,7 +5935,10 @@ function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane); if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); const gitMirror = record(target.gitMirror); const gitMirrorEgressProxy = record(gitMirror.egressProxy); - if (stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode") !== "node-global") throw new Error(`gitMirror.egressProxy.mode must be node-global for node=${spec.nodeId} lane=${spec.lane}`); - if (gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`); + const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode"); + if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global or direct for node=${spec.nodeId} lane=${spec.lane}`); + const nodeEgressProxy = gitMirrorEgressMode === "direct" + ? { mode: "direct" as const, required: false as const } + : nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`); + if (gitMirrorEgressMode === "node-global" && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`); const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport"); const source = record(target.source); const gitops = record(target.gitops);