From d25580c57cd0077203212eb44593eb9f1fca92fb Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Sun, 21 Jun 2026 12:52:36 +0800 Subject: [PATCH] fix: route D601 git mirror flush through YAML proxy (#568) Co-authored-by: Codex --- scripts/src/hwlab-node-control-plane.ts | 46 ++++++++++++++++++++----- scripts/src/hwlab-node-impl.ts | 26 ++++++++++++++ 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 54c91cb9..8907f751 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -1202,48 +1202,77 @@ NODE } function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { - if (target.gitMirror.egressProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub SSH no longer falls back to localhost`); - if (target.gitMirror.egressProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`); + const gitMirrorProxy = target.gitMirror.egressProxy; + if (gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub SSH no longer falls back to localhost`); + if (gitMirrorProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`); if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub SSH no longer falls back to localhost`); const proxyHost = `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`; const proxyPort = node.egressProxy.port; const proxyUrl = `http://${proxyHost}:${proxyPort}`; const noProxy = node.egressProxy.noProxy.join(","); + const proxySummary = `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`; + const proxyCommand = `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`; return [ "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", + `printf '%s\\n' ${shQuote(proxySummary)} >&2`, "cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'", "#!/usr/bin/env node", "const net = require('node:net');", "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", "const targetPort = Number.parseInt(targetPortRaw || '', 10);", - "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);", + "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {", + " console.error('hwlab git-mirror proxy-connect: invalid ProxyCommand arguments');", + " process.exit(64);", + "}", + "let settled = false;", + "let tunnelEstablished = false;", + "function finish(code, message) {", + " if (settled) return;", + " settled = true;", + " if (message) console.error('hwlab git-mirror proxy-connect: ' + message);", + " process.exit(code);", + "}", "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", "let buffer = Buffer.alloc(0);", - "socket.setTimeout(10000, () => { socket.destroy(); process.exit(65); });", + "socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });", "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", - "socket.on('error', () => process.exit(66));", + "socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));", + "socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });", "function onData(chunk) {", " buffer = Buffer.concat([buffer, chunk]);", " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", " if (headerEnd === -1 && buffer.length < 8192) return;", + " if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }", " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", " const statusLine = head.split('\\r\\n', 1)[0] || '';", " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", - " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); process.exit(67); }", + " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {", + " const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);", + " socket.destroy();", + " finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);", + " return;", + " }", " socket.off('data', onData);", " socket.setTimeout(0);", + " tunnelEstablished = true;", " const rest = buffer.slice(headerEnd + 4);", " if (rest.length) process.stdout.write(rest);", + " process.stdin.on('error', () => {});", + " process.stdout.on('error', () => {});", " process.stdin.pipe(socket);", " socket.pipe(process.stdout);", "}", "socket.on('data', onData);", - "socket.on('close', () => process.exit(0));", "NODE_PROXY", "chmod 0700 /tmp/hwlab-github-proxy-connect.cjs", + "cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'", + "#!/bin/sh", + `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`, + "SH_PROXY", + "chmod 0700 /tmp/hwlab-git-ssh-proxy.sh", `export HTTP_PROXY=${shQuote(proxyUrl)}`, `export HTTPS_PROXY=${shQuote(proxyUrl)}`, `export ALL_PROXY=${shQuote(proxyUrl)}`, @@ -1252,7 +1281,8 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT `export all_proxy=${shQuote(proxyUrl)}`, `export NO_PROXY=${shQuote(noProxy)}`, `export no_proxy=${shQuote(noProxy)}`, - `export GIT_SSH_COMMAND=${shQuote(`ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p'`)}`, + "export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh", + "unset GIT_SSH_COMMAND", `repository=${shQuote(target.source.repository)}`, `source_branch=${shQuote(target.source.branch)}`, `gitops_branch=${shQuote(target.gitops.branch)}`, diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index aec3c82d..fc29cbce 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -237,7 +237,9 @@ interface NodeRuntimeGitMirrorTargetSpec { serviceReadName: string; serviceWriteName: string; cachePvcName: string; + cachePvcStorage: string; cacheHostPath: string | null; + servicePort: number; secretName: string; syncConfigMapName: string; syncJobPrefix: string; @@ -2995,7 +2997,9 @@ function nodeRuntimeGitMirrorRetryableFailure( retryMaxAttempts: number, ): Record | null { const text = `${result.stdout}\n${result.stderr}`; + const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text); const retryable = partialSuccess !== null + || proxyConnectFailure || /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text); if (!retryable) return null; const retryLabel = `${attempt}/${retryMaxAttempts}`; @@ -3011,6 +3015,8 @@ function nodeRuntimeGitMirrorRetryableFailure( nextRetryDelayMs: exhausted ? null : nodeRuntimeGitMirrorRetryDelayMs(attempt), reason: partialSuccess !== null ? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback." + : proxyConnectFailure + ? "Git mirror job hit a retryable YAML-first proxy CONNECT failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted." : "Git mirror job hit a retryable upstream GitHub SSH/fetch failure. Standard git-mirror stops without host workspace fallback.", refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), next: exhausted @@ -4101,6 +4107,7 @@ function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceC " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", "};", "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;", + "if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;", "fs.writeFileSync(path, YAML.stringify(doc));", "NODE", "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", @@ -4184,6 +4191,7 @@ function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCo " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", "};", "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;", + "if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;", "fs.writeFileSync(path, YAML.stringify(doc));", "NODE", "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", @@ -5145,6 +5153,15 @@ function nodeRuntimePipelinePostprocessScript(): string[] { function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record { const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http); + const gitMirror = nodeRuntimeGitMirrorTarget(spec); + const renderGitMirror = { + ...gitMirror, + egressProxy: { + ...gitMirror.egressProxy, + mode: "node-global", + required: true, + }, + }; return { nodeId: spec.nodeId, lane: spec.lane, @@ -5161,6 +5178,7 @@ function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record { const trimmed = text.trim(); if (trimmed.length === 0) return {};