fix: route D601 git mirror flush through YAML proxy (#568)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
@@ -1202,48 +1202,77 @@ NODE
|
||||
}
|
||||
|
||||
function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
|
||||
if (target.gitMirror.egressProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub SSH no longer falls back to localhost`);
|
||||
if (target.gitMirror.egressProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
|
||||
const gitMirrorProxy = target.gitMirror.egressProxy;
|
||||
if (gitMirrorProxy?.mode !== "node-global") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global; git-mirror GitHub SSH no longer falls back to localhost`);
|
||||
if (gitMirrorProxy.required && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
|
||||
if (node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub SSH no longer falls back to localhost`);
|
||||
const proxyHost = `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`;
|
||||
const proxyPort = node.egressProxy.port;
|
||||
const proxyUrl = `http://${proxyHost}:${proxyPort}`;
|
||||
const noProxy = node.egressProxy.noProxy.join(",");
|
||||
const proxySummary = `git-mirror-egress-proxy client=${node.egressProxy.clientName} mode=node-global required=${gitMirrorProxy.required ? "true" : "false"} host=${proxyHost} port=${proxyPort} ssh=GIT_SSH-wrapper source=yaml`;
|
||||
const proxyCommand = `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
|
||||
return [
|
||||
"mkdir -p /root/.ssh",
|
||||
"cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa",
|
||||
"chmod 0400 /root/.ssh/id_rsa",
|
||||
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
|
||||
"cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'",
|
||||
"#!/usr/bin/env node",
|
||||
"const net = require('node:net');",
|
||||
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
|
||||
"const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
|
||||
"const targetPort = Number.parseInt(targetPortRaw || '', 10);",
|
||||
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);",
|
||||
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {",
|
||||
" console.error('hwlab git-mirror proxy-connect: invalid ProxyCommand arguments');",
|
||||
" process.exit(64);",
|
||||
"}",
|
||||
"let settled = false;",
|
||||
"let tunnelEstablished = false;",
|
||||
"function finish(code, message) {",
|
||||
" if (settled) return;",
|
||||
" settled = true;",
|
||||
" if (message) console.error('hwlab git-mirror proxy-connect: ' + message);",
|
||||
" process.exit(code);",
|
||||
"}",
|
||||
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
|
||||
"let buffer = Buffer.alloc(0);",
|
||||
"socket.setTimeout(10000, () => { socket.destroy(); process.exit(65); });",
|
||||
"socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });",
|
||||
"socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));",
|
||||
"socket.on('error', () => process.exit(66));",
|
||||
"socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));",
|
||||
"socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });",
|
||||
"function onData(chunk) {",
|
||||
" buffer = Buffer.concat([buffer, chunk]);",
|
||||
" const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
|
||||
" if (headerEnd === -1 && buffer.length < 8192) return;",
|
||||
" if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }",
|
||||
" const head = buffer.slice(0, headerEnd + 4).toString('latin1');",
|
||||
" const statusLine = head.split('\\r\\n', 1)[0] || '';",
|
||||
" const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
|
||||
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); process.exit(67); }",
|
||||
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {",
|
||||
" const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);",
|
||||
" socket.destroy();",
|
||||
" finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);",
|
||||
" return;",
|
||||
" }",
|
||||
" socket.off('data', onData);",
|
||||
" socket.setTimeout(0);",
|
||||
" tunnelEstablished = true;",
|
||||
" const rest = buffer.slice(headerEnd + 4);",
|
||||
" if (rest.length) process.stdout.write(rest);",
|
||||
" process.stdin.on('error', () => {});",
|
||||
" process.stdout.on('error', () => {});",
|
||||
" process.stdin.pipe(socket);",
|
||||
" socket.pipe(process.stdout);",
|
||||
"}",
|
||||
"socket.on('data', onData);",
|
||||
"socket.on('close', () => process.exit(0));",
|
||||
"NODE_PROXY",
|
||||
"chmod 0700 /tmp/hwlab-github-proxy-connect.cjs",
|
||||
"cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'",
|
||||
"#!/bin/sh",
|
||||
`exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
|
||||
"SH_PROXY",
|
||||
"chmod 0700 /tmp/hwlab-git-ssh-proxy.sh",
|
||||
`export HTTP_PROXY=${shQuote(proxyUrl)}`,
|
||||
`export HTTPS_PROXY=${shQuote(proxyUrl)}`,
|
||||
`export ALL_PROXY=${shQuote(proxyUrl)}`,
|
||||
@@ -1252,7 +1281,8 @@ function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneT
|
||||
`export all_proxy=${shQuote(proxyUrl)}`,
|
||||
`export NO_PROXY=${shQuote(noProxy)}`,
|
||||
`export no_proxy=${shQuote(noProxy)}`,
|
||||
`export GIT_SSH_COMMAND=${shQuote(`ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p'`)}`,
|
||||
"export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh",
|
||||
"unset GIT_SSH_COMMAND",
|
||||
`repository=${shQuote(target.source.repository)}`,
|
||||
`source_branch=${shQuote(target.source.branch)}`,
|
||||
`gitops_branch=${shQuote(target.gitops.branch)}`,
|
||||
|
||||
@@ -237,7 +237,9 @@ interface NodeRuntimeGitMirrorTargetSpec {
|
||||
serviceReadName: string;
|
||||
serviceWriteName: string;
|
||||
cachePvcName: string;
|
||||
cachePvcStorage: string;
|
||||
cacheHostPath: string | null;
|
||||
servicePort: number;
|
||||
secretName: string;
|
||||
syncConfigMapName: string;
|
||||
syncJobPrefix: string;
|
||||
@@ -2995,7 +2997,9 @@ function nodeRuntimeGitMirrorRetryableFailure(
|
||||
retryMaxAttempts: number,
|
||||
): Record<string, unknown> | null {
|
||||
const text = `${result.stdout}\n${result.stderr}`;
|
||||
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
|
||||
const retryable = partialSuccess !== null
|
||||
|| proxyConnectFailure
|
||||
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
|
||||
if (!retryable) return null;
|
||||
const retryLabel = `${attempt}/${retryMaxAttempts}`;
|
||||
@@ -3011,6 +3015,8 @@ function nodeRuntimeGitMirrorRetryableFailure(
|
||||
nextRetryDelayMs: exhausted ? null : nodeRuntimeGitMirrorRetryDelayMs(attempt),
|
||||
reason: partialSuccess !== null
|
||||
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
|
||||
: proxyConnectFailure
|
||||
? "Git mirror job hit a retryable YAML-first proxy CONNECT failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted."
|
||||
: "Git mirror job hit a retryable upstream GitHub SSH/fetch failure. Standard git-mirror stops without host workspace fallback.",
|
||||
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
||||
next: exhausted
|
||||
@@ -4101,6 +4107,7 @@ function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceC
|
||||
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
|
||||
"};",
|
||||
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
|
||||
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
|
||||
"fs.writeFileSync(path, YAML.stringify(doc));",
|
||||
"NODE",
|
||||
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
|
||||
@@ -4184,6 +4191,7 @@ function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCo
|
||||
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
|
||||
"};",
|
||||
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
|
||||
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
|
||||
"fs.writeFileSync(path, YAML.stringify(doc));",
|
||||
"NODE",
|
||||
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
|
||||
@@ -5145,6 +5153,15 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
|
||||
function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
||||
const gitSshProxy = httpProxyEndpoint(spec.networkProfile.proxy.http);
|
||||
const gitMirror = nodeRuntimeGitMirrorTarget(spec);
|
||||
const renderGitMirror = {
|
||||
...gitMirror,
|
||||
egressProxy: {
|
||||
...gitMirror.egressProxy,
|
||||
mode: "node-global",
|
||||
required: true,
|
||||
},
|
||||
};
|
||||
return {
|
||||
nodeId: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
@@ -5161,6 +5178,7 @@ function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, un
|
||||
gitUrl: spec.gitUrl,
|
||||
gitReadUrl: spec.gitReadUrl,
|
||||
gitWriteUrl: spec.gitWriteUrl,
|
||||
gitMirror: renderGitMirror,
|
||||
networkProfileId: spec.networkProfileId,
|
||||
downloadProfileId: spec.downloadProfileId,
|
||||
gitSshProxyHost: gitSshProxy?.host,
|
||||
@@ -5225,6 +5243,7 @@ function httpProxyEndpoint(value: string): { host: string; port: number } | null
|
||||
|
||||
function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] {
|
||||
return [
|
||||
`${renderDir}/devops-infra/git-mirror.yaml`,
|
||||
`${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`,
|
||||
`${renderDir}/${spec.tektonDir}/rbac.yaml`,
|
||||
`${renderDir}/${spec.tektonDir}/pipeline.yaml`,
|
||||
@@ -5884,7 +5903,9 @@ function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitM
|
||||
serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"),
|
||||
serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"),
|
||||
cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"),
|
||||
cachePvcStorage: stringValue(gitMirror.cachePvcStorage, "gitMirror.cachePvcStorage"),
|
||||
cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"),
|
||||
servicePort: positiveIntegerValue(gitMirror.servicePort, "gitMirror.servicePort"),
|
||||
secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"),
|
||||
syncConfigMapName: stringValue(gitMirror.syncConfigMapName, "gitMirror.syncConfigMapName"),
|
||||
syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"),
|
||||
@@ -10116,6 +10137,11 @@ function optionalStringValue(value: unknown, path: string): string | null {
|
||||
return value;
|
||||
}
|
||||
|
||||
function positiveIntegerValue(value: unknown, path: string): number {
|
||||
if (!Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseJsonObject(text: string): Record<string, unknown> {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.length === 0) return {};
|
||||
|
||||
Reference in New Issue
Block a user