mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(hwlab): allow direct ssh git mirror egress (#807)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-24 09:40:30 +08:00
committed by GitHub
parent 2a72892f70
commit fc7047d467
3 changed files with 90 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/hwlab-node-control-plane.yaml
+2 -2
View File
@@ -114,8 +114,8 @@ targets:
readUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git
writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git
egressProxy:
mode: node-global
required: true
mode: direct
required: false
githubTransport:
mode: ssh
tekton:
scripts/src/hwlab-node-control-plane.ts
+60 -51
View File
@@ -52,7 +52,7 @@ interface ControlPlaneEgressProxySpec {
}
interface ControlPlaneGitMirrorEgressProxySpec {
mode: "node-global";
mode: "node-global" | "direct";
required: boolean;
}
@@ -859,10 +859,10 @@ function egressProxySpec(raw: Record<string, unknown>, path: string): ControlPla
function gitMirrorEgressProxySpec(raw: Record<string, unknown>, path: string): ControlPlaneGitMirrorEgressProxySpec {
const mode = stringField(raw, "mode", path);
if (mode !== "node-global") throw new Error(`${path}.mode must be node-global`);
if (mode !== "node-global" && mode !== "direct") throw new Error(`${path}.mode must be node-global or direct`);
return {
mode,
required: raw.required === undefined ? true : booleanField(raw, "required", path),
required: raw.required === undefined ? mode !== "direct" : booleanField(raw, "required", path),
};
}
@@ -1284,58 +1284,37 @@ NODE
function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
const gitMirrorProxy = target.gitMirror.egressProxy;
const transport = target.gitMirror.githubTransport;
if (gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub transport no longer falls back to localhost`);
if (gitMirrorProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub transport no longer falls back to localhost`);
const proxyHost = `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`;
const proxyPort = node.egressProxy.port;
const proxyUrl = `http://${proxyHost}:${proxyPort}`;
const noProxy = node.egressProxy.noProxy.join(",");
const proxySummary = transport.mode === "https"
? `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=https proxy=HTTP_PROXY authSecret=${transport.tokenSecretName} authKey=${transport.tokenSecretKey} authSourceRef=${transport.tokenSourceRef} source=yaml`
: `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`;
const proxyCommand = `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
const useDirect = gitMirrorProxy?.mode === "direct";
const proxyRequired = gitMirrorProxy?.required === true;
if (!useDirect && gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global or direct`);
if (!useDirect && proxyRequired && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
if (!useDirect && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub transport cannot use node-global proxy`);
const proxyHost = useDirect || node.egressProxy === null ? "" : `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`;
const proxyPort = useDirect || node.egressProxy === null ? 0 : node.egressProxy.port;
const proxyUrl = useDirect ? "" : `http://${proxyHost}:${proxyPort}`;
const noProxy = useDirect || node.egressProxy === null ? "" : node.egressProxy.noProxy.join(",");
const proxySummary = useDirect
? `git-mirror-egress-proxy mode=direct required=false transport=${transport.mode} source=yaml`
: transport.mode === "https"
? `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=node-global required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=https proxy=HTTP_PROXY authSecret=${transport.tokenSecretName} authKey=${transport.tokenSecretKey} authSourceRef=${transport.tokenSourceRef} source=yaml`
: `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=node-global required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper source=yaml`;
const proxyCommand = useDirect ? "" : `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
const common = [
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
`export HTTP_PROXY=${shQuote(proxyUrl)}`,
`export HTTPS_PROXY=${shQuote(proxyUrl)}`,
`export ALL_PROXY=${shQuote(proxyUrl)}`,
`export http_proxy=${shQuote(proxyUrl)}`,
`export https_proxy=${shQuote(proxyUrl)}`,
`export all_proxy=${shQuote(proxyUrl)}`,
`export NO_PROXY=${shQuote(noProxy)}`,
`export no_proxy=${shQuote(noProxy)}`,
useDirect ? "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy" : `export HTTP_PROXY=${shQuote(proxyUrl)}`,
useDirect ? "export NO_PROXY='*'" : `export HTTPS_PROXY=${shQuote(proxyUrl)}`,
useDirect ? "export no_proxy='*'" : `export ALL_PROXY=${shQuote(proxyUrl)}`,
useDirect ? "" : `export http_proxy=${shQuote(proxyUrl)}`,
useDirect ? "" : `export https_proxy=${shQuote(proxyUrl)}`,
useDirect ? "" : `export all_proxy=${shQuote(proxyUrl)}`,
useDirect ? "" : `export NO_PROXY=${shQuote(noProxy)}`,
useDirect ? "" : `export no_proxy=${shQuote(noProxy)}`,
`repository=${shQuote(target.source.repository)}`,
`source_branch=${shQuote(target.source.branch)}`,
`gitops_branch=${shQuote(target.gitops.branch)}`,
"repo=\"/cache/${repository}.git\"",
];
if (transport.mode === "https") {
return [
...common,
"if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo 'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env' >&2; exit 64; fi",
"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'",
"#!/bin/sh",
"case \"$1\" in",
" *Username*) printf '%s\\n' \"${GITHUB_USERNAME:-x-access-token}\" ;;",
" *Password*) printf '%s\\n' \"$GITHUB_TOKEN\" ;;",
" *) printf '\\n' ;;",
"esac",
"SH_ASKPASS",
"chmod 0700 /tmp/hwlab-git-askpass.sh",
`export GITHUB_USERNAME=${shQuote(transport.username)}`,
"export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh",
"export GIT_TERMINAL_PROMPT=0",
"unset GIT_SSH",
"unset GIT_SSH_COMMAND",
"remote=\"https://github.com/${repository}.git\"",
].join("\n");
}
return [
"mkdir -p /root/.ssh",
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
"chmod 0400 /root/.ssh/id_rsa",
...common,
].filter(Boolean);
const proxyConnectBlock = useDirect ? [] : [
"cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'",
"#!/usr/bin/env node",
"const net = require('node:net');",
@@ -1387,15 +1366,45 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT
"socket.on('data', onData);",
"NODE_PROXY",
"chmod 0700 /tmp/hwlab-github-proxy-connect.cjs",
];
if (transport.mode === "https") {
return [
...common,
"if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo 'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env' >&2; exit 64; fi",
"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'",
"#!/bin/sh",
"case \"$1\" in",
" *Username*) printf '%s\\n' \"${GITHUB_USERNAME:-x-access-token}\" ;;",
" *Password*) printf '%s\\n' \"$GITHUB_TOKEN\" ;;",
" *) printf '\\n' ;;",
"esac",
"SH_ASKPASS",
"chmod 0700 /tmp/hwlab-git-askpass.sh",
`export GITHUB_USERNAME=${shQuote(transport.username)}`,
"export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh",
"export GIT_TERMINAL_PROMPT=0",
"unset GIT_SSH",
"unset GIT_SSH_COMMAND",
"remote=\"https://github.com/${repository}.git\"",
].join("\n");
}
return [
"mkdir -p /root/.ssh",
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
"chmod 0400 /root/.ssh/id_rsa",
...common,
...proxyConnectBlock,
"cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'",
"#!/bin/sh",
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
useDirect
? `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$@"`
: `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
"SH_PROXY",
"chmod 0700 /tmp/hwlab-git-ssh-proxy.sh",
"export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh",
"unset GIT_SSH_COMMAND",
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
].join("\n");
].filter(Boolean).join("\n");
}
function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
scripts/src/hwlab-node-impl.ts
+28 -17
View File
@@ -274,17 +274,19 @@ type NodeRuntimeGitMirrorGithubTransportSpec =
tokenSourceKey: string;
};
interface NodeRuntimeGitMirrorEgressProxySpec {
mode: "k8s-service-cluster-ip";
clientName: string;
namespace: string;
serviceName: string;
port: number;
sourceRef: string;
sourceKey: string;
sourceType: "subscription-url";
noProxy: string[];
}
type NodeRuntimeGitMirrorEgressProxySpec =
| { mode: "direct"; required: false }
| {
mode: "k8s-service-cluster-ip";
clientName: string;
namespace: string;
serviceName: string;
port: number;
sourceRef: string;
sourceKey: string;
sourceType: "subscription-url";
noProxy: string[];
};
const MASTER_ADMIN_API_KEY_KEY = "api-key";
const BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY = "password-hash";
@@ -3386,6 +3388,7 @@ function nodeRuntimeGitMirrorRetryableFailure(
): Record<string, unknown> | null {
const text = `${result.stdout}\n${result.stderr}`;
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
const sshProxyBannerFailure = mirror.egressProxy.mode !== "direct" && /Connection timed out during banner exchange|Connection to UNKNOWN port 65535 timed out|kex_exchange_identification|Connection closed by remote host/iu.test(text);
const waitTimeoutFailure = result.exitCode === 45
|| result.exitCode === 124
|| /UNIDESK_SSH_RUNTIME_TIMEOUT|ssh-runtime-timeout|ssh\/tran operation exceeded|job,pod/iu.test(text);
@@ -3437,6 +3440,7 @@ function nodeRuntimeGitMirrorRetryableFailure(
}
const retryable = partialSuccess !== null
|| proxyConnectFailure
|| sshProxyBannerFailure
|| waitTimeoutFailure
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
if (!retryable) return null;
@@ -3455,8 +3459,8 @@ function nodeRuntimeGitMirrorRetryableFailure(
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
: waitTimeoutFailure
? "Git mirror job wait exceeded the controlled short-connection budget. Standard git-mirror retries with exponential backoff and stops when retry budget is exhausted."
: proxyConnectFailure
? "Git mirror job hit a retryable YAML-first proxy CONNECT failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted."
: proxyConnectFailure || sshProxyBannerFailure
? "Git mirror job hit a retryable YAML-first SSH-over-proxy failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted."
: `Git mirror job hit a retryable upstream GitHub ${mirror.githubTransport.mode} transport/fetch failure. Standard git-mirror stops without host workspace fallback.`,
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
@@ -3860,6 +3864,7 @@ function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec,
function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
const proxy = mirror.egressProxy;
if (proxy.mode === "direct") return [];
const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`;
const noProxy = proxy.noProxy.join(",");
return [
@@ -5930,7 +5935,10 @@ function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, un
const gitMirror = nodeRuntimeGitMirrorTarget(spec);
const renderGitMirror = {
...gitMirror,
egressProxy: {
egressProxy: gitMirror.egressProxy.mode === "direct" ? {
mode: "direct",
required: false,
} : {
...gitMirror.egressProxy,
mode: "node-global",
required: true,
@@ -6948,14 +6956,17 @@ function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitM
const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown);
const nodes = record(parsed.nodes);
const node = record(nodes[spec.nodeId]);
const nodeEgressProxy = nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`);
const targets = Array.isArray(parsed.targets) ? parsed.targets : [];
const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane);
if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`);
const gitMirror = record(target.gitMirror);
const gitMirrorEgressProxy = record(gitMirror.egressProxy);
if (stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode") !== "node-global") throw new Error(`gitMirror.egressProxy.mode must be node-global for node=${spec.nodeId} lane=${spec.lane}`);
if (gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode");
if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global or direct for node=${spec.nodeId} lane=${spec.lane}`);
const nodeEgressProxy = gitMirrorEgressMode === "direct"
? { mode: "direct" as const, required: false as const }
: nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`);
if (gitMirrorEgressMode === "node-global" && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport");
const source = record(target.source);
const gitops = record(target.gitops);