mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: route D601 git mirror flush through YAML proxy (#568)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-21 12:52:36 +08:00
committed by GitHub
parent c849c95815
commit d25580c57c
2 changed files with 64 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-control-plane.ts
+38 -8
View File
@@ -1202,48 +1202,77 @@ NODE
}
function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
if (target.gitMirror.egressProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub SSH no longer falls back to localhost`);
if (target.gitMirror.egressProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
const gitMirrorProxy = target.gitMirror.egressProxy;
if (gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub SSH no longer falls back to localhost`);
if (gitMirrorProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub SSH no longer falls back to localhost`);
const proxyHost = `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`;
const proxyPort = node.egressProxy.port;
const proxyUrl = `http://${proxyHost}:${proxyPort}`;
const noProxy = node.egressProxy.noProxy.join(",");
const proxySummary = `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`;
const proxyCommand = `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
return [
"mkdir -p /root/.ssh",
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
"chmod 0400 /root/.ssh/id_rsa",
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
"cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'",
"#!/usr/bin/env node",
"const net = require('node:net');",
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
"const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
"const targetPort = Number.parseInt(targetPortRaw || '', 10);",
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);",
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {",
" console.error('hwlab git-mirror proxy-connect: invalid ProxyCommand arguments');",
" process.exit(64);",
"}",
"let settled = false;",
"let tunnelEstablished = false;",
"function finish(code, message) {",
" if (settled) return;",
" settled = true;",
" if (message) console.error('hwlab git-mirror proxy-connect: ' + message);",
" process.exit(code);",
"}",
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
"let buffer = Buffer.alloc(0);",
"socket.setTimeout(10000, () => { socket.destroy(); process.exit(65); });",
"socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });",
"socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));",
"socket.on('error', () => process.exit(66));",
"socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));",
"socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });",
"function onData(chunk) {",
" buffer = Buffer.concat([buffer, chunk]);",
" const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
" if (headerEnd === -1 && buffer.length < 8192) return;",
" if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }",
" const head = buffer.slice(0, headerEnd + 4).toString('latin1');",
" const statusLine = head.split('\\r\\n', 1)[0] || '';",
" const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); process.exit(67); }",
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {",
" const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);",
" socket.destroy();",
" finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);",
" return;",
" }",
" socket.off('data', onData);",
" socket.setTimeout(0);",
" tunnelEstablished = true;",
" const rest = buffer.slice(headerEnd + 4);",
" if (rest.length) process.stdout.write(rest);",
" process.stdin.on('error', () => {});",
" process.stdout.on('error', () => {});",
" process.stdin.pipe(socket);",
" socket.pipe(process.stdout);",
"}",
"socket.on('data', onData);",
"socket.on('close', () => process.exit(0));",
"NODE_PROXY",
"chmod 0700 /tmp/hwlab-github-proxy-connect.cjs",
"cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'",
"#!/bin/sh",
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
"SH_PROXY",
"chmod 0700 /tmp/hwlab-git-ssh-proxy.sh",
`export HTTP_PROXY=${shQuote(proxyUrl)}`,
`export HTTPS_PROXY=${shQuote(proxyUrl)}`,
`export ALL_PROXY=${shQuote(proxyUrl)}`,
@@ -1252,7 +1281,8 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT
`export all_proxy=${shQuote(proxyUrl)}`,
`export NO_PROXY=${shQuote(noProxy)}`,
`export no_proxy=${shQuote(noProxy)}`,
`export GIT_SSH_COMMAND=${shQuote(`ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p'`)}`,
"export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh",
"unset GIT_SSH_COMMAND",
`repository=${shQuote(target.source.repository)}`,
`source_branch=${shQuote(target.source.branch)}`,
`gitops_branch=${shQuote(target.gitops.branch)}`,
scripts/src/hwlab-node-impl.ts
+26
View File
@@ -237,7 +237,9 @@ interface NodeRuntimeGitMirrorTargetSpec {
serviceReadName: string;
serviceWriteName: string;
cachePvcName: string;
cachePvcStorage: string;
cacheHostPath: string | null;
servicePort: number;
secretName: string;
syncConfigMapName: string;
syncJobPrefix: string;
@@ -2995,7 +2997,9 @@ function nodeRuntimeGitMirrorRetryableFailure(
retryMaxAttempts: number,
): Record<string, unknown> | null {
const text = `${result.stdout}\n${result.stderr}`;
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
const retryable = partialSuccess !== null
|| proxyConnectFailure
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
if (!retryable) return null;
const retryLabel = `${attempt}/${retryMaxAttempts}`;
@@ -3011,6 +3015,8 @@ function nodeRuntimeGitMirrorRetryableFailure(
nextRetryDelayMs: exhausted ? null : nodeRuntimeGitMirrorRetryDelayMs(attempt),
reason: partialSuccess !== null
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
: proxyConnectFailure
? "Git mirror job hit a retryable YAML-first proxy CONNECT failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted."
: "Git mirror job hit a retryable upstream GitHub SSH/fetch failure. Standard git-mirror stops without host workspace fallback.",
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
next: exhausted
@@ -4101,6 +4107,7 @@ function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceC
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
"};",
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
"fs.writeFileSync(path, YAML.stringify(doc));",
"NODE",
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
@@ -4184,6 +4191,7 @@ function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCo
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
"};",
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
"fs.writeFileSync(path, YAML.stringify(doc));",
"NODE",
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
@@ -5145,6 +5153,15 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http);
const gitMirror = nodeRuntimeGitMirrorTarget(spec);
const renderGitMirror = {
...gitMirror,
egressProxy: {
...gitMirror.egressProxy,
mode: "node-global",
required: true,
},
};
return {
nodeId: spec.nodeId,
lane: spec.lane,
@@ -5161,6 +5178,7 @@ function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, un
gitUrl: spec.gitUrl,
gitReadUrl: spec.gitReadUrl,
gitWriteUrl: spec.gitWriteUrl,
gitMirror: renderGitMirror,
networkProfileId: spec.networkProfileId,
downloadProfileId: spec.downloadProfileId,
gitSshProxyHost: gitSshProxy?.host,
@@ -5225,6 +5243,7 @@ function httpProxyEndpoint(value: string): { host: string; port: number } | null
function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] {
return [
`${renderDir}/devops-infra/git-mirror.yaml`,
`${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`,
`${renderDir}/${spec.tektonDir}/rbac.yaml`,
`${renderDir}/${spec.tektonDir}/pipeline.yaml`,
@@ -5884,7 +5903,9 @@ function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitM
serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"),
cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"),
cachePvcStorage: stringValue(gitMirror.cachePvcStorage, "gitMirror.cachePvcStorage"),
cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"),
servicePort: positiveIntegerValue(gitMirror.servicePort, "gitMirror.servicePort"),
secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"),
syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"),
syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"),
@@ -10116,6 +10137,11 @@ function optionalStringValue(value: unknown, path: string): string | null {
return value;
}
function positiveIntegerValue(value: unknown, path: string): number {
if (!Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
return value;
}
function parseJsonObject(text: string): Record<string, unknown> {
const trimmed = text.trim();
if (trimmed.length === 0) return {};