fix: load git mirror scripts from files

This commit is contained in:
Codex
2026-07-03 16:02:41 +00:00
parent e973b73dbc
commit edb6fe7485
5 changed files with 299 additions and 285 deletions
@@ -0,0 +1,129 @@
import { readFileSync } from "node:fs";
import { rootPath } from "./config";
import type { ControlPlaneNodeSpec, ControlPlaneTargetSpec } from "./hwlab-node-control-plane-model";
const SCRIPT_ROOT = "scripts/src/hwlab-node-control-plane";
export function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
return renderShellTemplate("git-mirror-sync.sh", {
GIT_MIRROR_PROXY_PRELUDE: gitMirrorProxyPrelude(node, target),
});
}
export function gitMirrorFlushShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
return renderShellTemplate("git-mirror-flush.sh", {
GIT_MIRROR_PROXY_PRELUDE: gitMirrorProxyPrelude(node, target),
});
}
function renderShellTemplate(fileName: string, replacements: Record<string, string>): string {
let script = readScript(fileName);
for (const [key, value] of Object.entries(replacements)) script = script.replaceAll(`__${key}__`, value.trimEnd());
if (/__[A-Z0-9_]+__/u.test(script)) throw new Error(`unresolved git-mirror script template marker in ${fileName}`);
return script.endsWith("\n") ? script : `${script}\n`;
}
function readScript(fileName: string): string {
return readFileSync(rootPath(SCRIPT_ROOT, fileName), "utf8");
}
function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
const gitMirrorProxy = target.gitMirror.egressProxy;
const transport = target.gitMirror.githubTransport;
const useDirect = gitMirrorProxy?.mode === "direct";
const proxyRequired = gitMirrorProxy?.required === true;
if (!useDirect && gitMirrorProxy?.mode !== "node-global" && gitMirrorProxy?.mode !== "host-route") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global, host-route, or direct`);
if (!useDirect && proxyRequired && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
if (!useDirect && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub transport cannot use node-global proxy`);
if (!useDirect && gitMirrorProxy?.mode === "node-global" && node.egressProxy?.mode !== "k8s-service-cluster-ip") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode=node-global requires nodes.${node.id}.egressProxy.mode=k8s-service-cluster-ip`);
if (!useDirect && gitMirrorProxy?.mode === "host-route" && node.egressProxy?.mode !== "host-route") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode=host-route requires nodes.${node.id}.egressProxy.mode=host-route`);
const hostRouteUrl = !useDirect && node.egressProxy?.mode === "host-route" ? new URL(node.egressProxy.proxyUrl) : null;
const proxyHost = useDirect || node.egressProxy === null ? "" : node.egressProxy.mode === "host-route" ? hostRouteUrl!.hostname : `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`;
const proxyPort = useDirect || node.egressProxy === null ? 0 : node.egressProxy.mode === "host-route" ? Number(hostRouteUrl!.port) : node.egressProxy.port;
const proxyUrl = useDirect || node.egressProxy === null ? "" : node.egressProxy.mode === "host-route" ? node.egressProxy.proxyUrl : `http://${proxyHost}:${proxyPort}`;
const noProxy = useDirect || node.egressProxy === null ? "" : node.egressProxy.noProxy.join(",");
const proxySource = useDirect || node.egressProxy === null
? "sourceType=direct sourceRef=- sourceFingerprint=-"
: node.egressProxy.mode === "host-route"
? `sourceType=host-route hostProxyConfigRef=${node.egressProxy.hostProxyConfigRef} sourceFingerprint=-`
: `sourceType=${node.egressProxy.sourceType} sourceRef=${node.egressProxy.sourceRef} sourceFingerprint=${node.egressProxy.sourceFingerprint ?? "-"}`;
const proxySummary = useDirect
? `git-mirror-egress-proxy mode=direct required=false transport=${transport.mode} source=yaml`
: transport.mode === "https"
? `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=${node.egressProxy?.mode} required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=https proxy=HTTP_PROXY authSecret=${transport.tokenSecretName} authKey=${transport.tokenSecretKey} authSourceRef=${transport.tokenSourceRef} ${proxySource} source=yaml`
: `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=${node.egressProxy?.mode} required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper ${proxySource} source=yaml`;
const common = [
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
useDirect ? "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy" : `export HTTP_PROXY=${shQuote(proxyUrl)}`,
useDirect ? "export NO_PROXY='*'" : `export HTTPS_PROXY=${shQuote(proxyUrl)}`,
useDirect ? "export no_proxy='*'" : `export ALL_PROXY=${shQuote(proxyUrl)}`,
useDirect ? "" : `export http_proxy=${shQuote(proxyUrl)}`,
useDirect ? "" : `export https_proxy=${shQuote(proxyUrl)}`,
useDirect ? "" : `export all_proxy=${shQuote(proxyUrl)}`,
useDirect ? "" : `export NO_PROXY=${shQuote(noProxy)}`,
useDirect ? "" : `export no_proxy=${shQuote(noProxy)}`,
`repository=${shQuote(target.source.repository)}`,
`source_branch=${shQuote(target.source.branch)}`,
`source_stage_ref_prefix=${shQuote(target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch))}`,
`gitops_branch=${shQuote(target.gitops.branch)}`,
"repo=\"/cache/${repository}.git\"",
].filter(Boolean);
if (transport.mode === "https") {
return [
...common,
"if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo 'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env' >&2; exit 64; fi",
"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'",
"#!/bin/sh",
"case \"$1\" in",
" *Username*) printf '%s\\n' \"${GITHUB_USERNAME:-x-access-token}\" ;;",
" *Password*) printf '%s\\n' \"$GITHUB_TOKEN\" ;;",
" *) printf '\\n' ;;",
"esac",
"SH_ASKPASS",
"chmod 0700 /tmp/hwlab-git-askpass.sh",
`export GITHUB_USERNAME=${shQuote(transport.username)}`,
"export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh",
"export GIT_TERMINAL_PROMPT=0",
"unset GIT_SSH",
"unset GIT_SSH_COMMAND",
"remote=\"https://github.com/${repository}.git\"",
].join("\n");
}
const proxyCommand = useDirect ? "" : `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
const privateKeyPath = transport.mode === "ssh" ? `/git-ssh/${transport.privateKeySecretKey}` : "/git-ssh/ssh-privatekey";
const knownHostsCopy = transport.mode === "ssh" && transport.knownHostsSecretKey !== null
? [`cp ${shQuote(`/git-ssh/${transport.knownHostsSecretKey}`)} /root/.ssh/known_hosts`, "chmod 0600 /root/.ssh/known_hosts"]
: [];
return [
"mkdir -p /root/.ssh",
`cp ${shQuote(privateKeyPath)} /root/.ssh/id_rsa`,
"chmod 0400 /root/.ssh/id_rsa",
...knownHostsCopy,
...common,
...proxyConnectBlock(useDirect),
"cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'",
"#!/bin/sh",
useDirect
? `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$@"`
: `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
"SH_PROXY",
"chmod 0700 /tmp/hwlab-git-ssh-proxy.sh",
"export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh",
"unset GIT_SSH_COMMAND",
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
].filter(Boolean).join("\n");
}
function proxyConnectBlock(useDirect: boolean): string[] {
if (useDirect) return [];
return [
"cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'",
readScript("git-mirror-proxy-connect.cjs").trimEnd(),
"NODE_PROXY",
"chmod 0700 /tmp/hwlab-github-proxy-connect.cjs",
];
}
function shQuote(value: string): string {
return `'${value.replace(/'/gu, `'"'"'`)}'`;
}
+1 -285
View File
@@ -78,6 +78,7 @@ import {
toolsImageDockerfile,
toolsImageStatus,
} from "./hwlab-node-control-plane-runtime";
import { gitMirrorFlushShell, gitMirrorSyncShell } from "./hwlab-node-control-plane-git-mirror-scripts";
import type { RenderedCliResult } from "./output";
import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets";
@@ -2551,287 +2552,6 @@ NODE
`;
}
function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
const gitMirrorProxy = target.gitMirror.egressProxy;
const transport = target.gitMirror.githubTransport;
const useDirect = gitMirrorProxy?.mode === "direct";
const proxyRequired = gitMirrorProxy?.required === true;
if (!useDirect && gitMirrorProxy?.mode !== "node-global" && gitMirrorProxy?.mode !== "host-route") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global, host-route, or direct`);
if (!useDirect && proxyRequired && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`);
if (!useDirect && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub transport cannot use node-global proxy`);
if (!useDirect && gitMirrorProxy?.mode === "node-global" && node.egressProxy?.mode !== "k8s-service-cluster-ip") {
throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode=node-global requires nodes.${node.id}.egressProxy.mode=k8s-service-cluster-ip`);
}
if (!useDirect && gitMirrorProxy?.mode === "host-route" && node.egressProxy?.mode !== "host-route") {
throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode=host-route requires nodes.${node.id}.egressProxy.mode=host-route`);
}
const hostRouteUrl = !useDirect && node.egressProxy?.mode === "host-route" ? new URL(node.egressProxy.proxyUrl) : null;
const proxyHost = useDirect || node.egressProxy === null
? ""
: node.egressProxy.mode === "host-route"
? hostRouteUrl!.hostname
: `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`;
const proxyPort = useDirect || node.egressProxy === null
? 0
: node.egressProxy.mode === "host-route"
? Number(hostRouteUrl!.port)
: node.egressProxy.port;
const proxyUrl = useDirect || node.egressProxy === null ? "" : node.egressProxy.mode === "host-route" ? node.egressProxy.proxyUrl : `http://${proxyHost}:${proxyPort}`;
const noProxy = useDirect || node.egressProxy === null ? "" : node.egressProxy.noProxy.join(",");
const proxySource = useDirect || node.egressProxy === null
? "sourceType=direct sourceRef=- sourceFingerprint=-"
: node.egressProxy.mode === "host-route"
? `sourceType=host-route hostProxyConfigRef=${node.egressProxy.hostProxyConfigRef} sourceFingerprint=-`
: `sourceType=${node.egressProxy.sourceType} sourceRef=${node.egressProxy.sourceRef} sourceFingerprint=${node.egressProxy.sourceFingerprint ?? "-"}`;
const proxySummary = useDirect
? `git-mirror-egress-proxy mode=direct required=false transport=${transport.mode} source=yaml`
: transport.mode === "https"
? `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=${node.egressProxy?.mode} required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=https proxy=HTTP_PROXY authSecret=${transport.tokenSecretName} authKey=${transport.tokenSecretKey} authSourceRef=${transport.tokenSourceRef} ${proxySource} source=yaml`
: `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=${node.egressProxy?.mode} required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper ${proxySource} source=yaml`;
const proxyCommand = useDirect ? "" : `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`;
const common = [
`printf '%s\\n' ${shQuote(proxySummary)} >&2`,
useDirect ? "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy" : `export HTTP_PROXY=${shQuote(proxyUrl)}`,
useDirect ? "export NO_PROXY='*'" : `export HTTPS_PROXY=${shQuote(proxyUrl)}`,
useDirect ? "export no_proxy='*'" : `export ALL_PROXY=${shQuote(proxyUrl)}`,
useDirect ? "" : `export http_proxy=${shQuote(proxyUrl)}`,
useDirect ? "" : `export https_proxy=${shQuote(proxyUrl)}`,
useDirect ? "" : `export all_proxy=${shQuote(proxyUrl)}`,
useDirect ? "" : `export NO_PROXY=${shQuote(noProxy)}`,
useDirect ? "" : `export no_proxy=${shQuote(noProxy)}`,
`repository=${shQuote(target.source.repository)}`,
`source_branch=${shQuote(target.source.branch)}`,
`source_stage_ref_prefix=${shQuote(target.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", target.source.branch))}`,
`gitops_branch=${shQuote(target.gitops.branch)}`,
"repo=\"/cache/${repository}.git\"",
].filter(Boolean);
const proxyConnectBlock = useDirect ? [] : [
"cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'",
"#!/usr/bin/env node",
"const net = require('node:net');",
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
"const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
"const targetPort = Number.parseInt(targetPortRaw || '', 10);",
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {",
" console.error('hwlab git-mirror proxy-connect: invalid ProxyCommand arguments');",
" process.exit(64);",
"}",
"let settled = false;",
"let tunnelEstablished = false;",
"function finish(code, message) {",
" if (settled) return;",
" settled = true;",
" if (message) console.error('hwlab git-mirror proxy-connect: ' + message);",
" process.exit(code);",
"}",
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
"let buffer = Buffer.alloc(0);",
"socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });",
"socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));",
"socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));",
"socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });",
"function onData(chunk) {",
" buffer = Buffer.concat([buffer, chunk]);",
" const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
" if (headerEnd === -1 && buffer.length < 8192) return;",
" if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }",
" const head = buffer.slice(0, headerEnd + 4).toString('latin1');",
" const statusLine = head.split('\\r\\n', 1)[0] || '';",
" const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {",
" const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);",
" socket.destroy();",
" finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);",
" return;",
" }",
" socket.off('data', onData);",
" socket.setTimeout(0);",
" tunnelEstablished = true;",
" const rest = buffer.slice(headerEnd + 4);",
" if (rest.length) process.stdout.write(rest);",
" process.stdin.on('error', () => {});",
" process.stdout.on('error', () => {});",
" process.stdin.pipe(socket);",
" socket.pipe(process.stdout);",
"}",
"socket.on('data', onData);",
"NODE_PROXY",
"chmod 0700 /tmp/hwlab-github-proxy-connect.cjs",
];
if (transport.mode === "https") {
return [
...common,
"if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo 'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env' >&2; exit 64; fi",
"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'",
"#!/bin/sh",
"case \"$1\" in",
" *Username*) printf '%s\\n' \"${GITHUB_USERNAME:-x-access-token}\" ;;",
" *Password*) printf '%s\\n' \"$GITHUB_TOKEN\" ;;",
" *) printf '\\n' ;;",
"esac",
"SH_ASKPASS",
"chmod 0700 /tmp/hwlab-git-askpass.sh",
`export GITHUB_USERNAME=${shQuote(transport.username)}`,
"export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh",
"export GIT_TERMINAL_PROMPT=0",
"unset GIT_SSH",
"unset GIT_SSH_COMMAND",
"remote=\"https://github.com/${repository}.git\"",
].join("\n");
}
const privateKeyPath = transport.mode === "ssh" ? `/git-ssh/${transport.privateKeySecretKey}` : "/git-ssh/ssh-privatekey";
const knownHostsCopy = transport.mode === "ssh" && transport.knownHostsSecretKey !== null
? [`cp ${shQuote(`/git-ssh/${transport.knownHostsSecretKey}`)} /root/.ssh/known_hosts`, "chmod 0600 /root/.ssh/known_hosts"]
: [];
return [
"mkdir -p /root/.ssh",
`cp ${shQuote(privateKeyPath)} /root/.ssh/id_rsa`,
"chmod 0400 /root/.ssh/id_rsa",
...knownHostsCopy,
...common,
...proxyConnectBlock,
"cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'",
"#!/bin/sh",
useDirect
? `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$@"`
: `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`,
"SH_PROXY",
"chmod 0700 /tmp/hwlab-git-ssh-proxy.sh",
"export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh",
"unset GIT_SSH_COMMAND",
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
].filter(Boolean).join("\n");
}
function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
return [
"#!/bin/sh",
"set -eu",
"started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
gitMirrorProxyPrelude(node, target),
"mkdir -p \"$(dirname \"$repo\")\"",
"if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then",
" git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"",
"else",
" rm -rf \"$repo\"",
" git init --bare \"$repo\"",
" git --git-dir=\"$repo\" remote add origin \"$remote\"",
"fi",
"git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true",
"git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true",
"git --git-dir=\"$repo\" config http.uploadpack true",
"git --git-dir=\"$repo\" config http.receivepack true",
"timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"",
"source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")",
"source_stage_ref=\"${source_stage_ref_prefix%/}/$source_sha\"",
"git --git-dir=\"$repo\" update-ref \"$source_stage_ref\" \"$source_sha\"",
"git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"",
"discarded_stale_gitops=false",
"if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then",
" github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
" local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
" if [ -z \"$local_gitops\" ] && [ -n \"$github_gitops\" ]; then",
" git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"",
" elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && git --git-dir=\"$repo\" merge-base --is-ancestor \"$local_gitops\" \"$github_gitops\"; then",
" git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"",
" elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && [ \"${UNIDESK_GIT_MIRROR_DISCARD_STALE_GITOPS:-false}\" = \"true\" ]; then",
" git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"",
" discarded_stale_gitops=true",
" printf '%s\\n' \"git-mirror sync: discarded stale local gitops ref local=${local_gitops} github=${github_gitops}\" >&2",
" fi",
"fi",
"git --git-dir=\"$repo\" update-server-info",
"export repository source_branch source_stage_ref gitops_branch started_at discarded_stale_gitops",
"node <<'NODE' | tee /cache/HWLAB.last-sync.json",
"const { execFileSync } = require('node:child_process');",
"const repository = process.env.repository;",
"const sourceBranch = process.env.source_branch;",
"const gitopsBranch = process.env.gitops_branch;",
"const repoPath = `/cache/${repository}.git`;",
"function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }",
"const localSource = rev(`refs/heads/${sourceBranch}`);",
"const githubSource = rev(`refs/mirror-stage/heads/${sourceBranch}`);",
"const sourceStageRef = process.env.source_stage_ref;",
"const sourceSnapshot = sourceStageRef ? rev(sourceStageRef) : null;",
"const localGitops = rev(`refs/heads/${gitopsBranch}`);",
"const githubGitops = rev(`refs/mirror-stage/heads/${gitopsBranch}`);",
"const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));",
"console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), sourceAuthority: 'git-mirror-snapshot', localSource, githubSource, sourceStageRef, sourceSnapshot, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));",
"NODE",
"cat /cache/HWLAB.last-sync.json",
"",
].join("\n");
}
function gitMirrorFlushShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
return [
"#!/bin/sh",
"set -eu",
"started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)",
gitMirrorProxyPrelude(node, target),
"test -d \"$repo/objects\"",
"git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"",
"local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
"push_status=skipped",
"push_exit=0",
"fetch_status=skipped",
"fetch_exit=0",
"fetch_attempt=0",
"fetch_max_attempts=5",
"if [ -n \"$local_gitops\" ]; then",
" set +e",
" timeout 240 git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"",
" push_exit=$?",
" set -e",
" if [ \"$push_exit\" = \"0\" ]; then",
" push_status=succeeded",
" fetch_retry_delay=1",
" while [ \"$fetch_attempt\" -lt \"$fetch_max_attempts\" ]; do",
" fetch_attempt=$((fetch_attempt + 1))",
" echo \"git-mirror post-push fetch attempt ${fetch_attempt}/${fetch_max_attempts}\" >&2",
" set +e",
" timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"",
" fetch_exit=$?",
" set -e",
" if [ \"$fetch_exit\" = \"0\" ]; then fetch_status=succeeded; break; fi",
" fetch_status=failed",
" if [ \"$fetch_attempt\" -lt \"$fetch_max_attempts\" ]; then",
" echo \"git-mirror post-push fetch retry ${fetch_attempt}/${fetch_max_attempts} failed exit=${fetch_exit}; backoff=${fetch_retry_delay}s\" >&2",
" sleep \"$fetch_retry_delay\"",
" if [ \"$fetch_retry_delay\" -lt 16 ]; then fetch_retry_delay=$((fetch_retry_delay * 2)); fi",
" fi",
" done",
" else",
" push_status=failed",
" fi",
"fi",
"github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)",
"pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi",
"status=succeeded",
"partial_success=",
"degraded_reason=",
"exit_code=0",
"if [ \"$push_status\" = \"failed\" ]; then",
" status=failed",
" degraded_reason=git-mirror-push-failed",
" exit_code=$push_exit",
"elif [ \"$push_status\" = \"succeeded\" ] && [ \"$fetch_status\" = \"failed\" ]; then",
" status=partial-success",
" partial_success=push-succeeded-fetch-failed",
" degraded_reason=git-mirror-post-push-fetch-failed",
" exit_code=44",
"fi",
"export repository gitops_branch started_at local_gitops github_gitops pending push_status push_exit fetch_status fetch_exit fetch_attempt fetch_max_attempts status partial_success degraded_reason",
"node <<'NODE' | tee /cache/HWLAB.last-flush.json",
"const payload = { event: 'git-mirror-flush', repo: process.env.repository, status: process.env.status || 'failed', partialSuccess: process.env.partial_success || null, degradedReason: process.env.degraded_reason || null, startedAt: process.env.started_at, flushedAt: new Date().toISOString(), gitopsBranch: process.env.gitops_branch, localGitops: process.env.local_gitops || null, githubGitops: process.env.github_gitops || null, pendingFlush: process.env.pending === 'true', stages: { push: process.env.push_status || null, pushExitCode: Number.parseInt(process.env.push_exit || '0', 10), postPushFetch: process.env.fetch_status || null, postPushFetchExitCode: Number.parseInt(process.env.fetch_exit || '0', 10), postPushFetchAttempts: Number.parseInt(process.env.fetch_attempt || '0', 10), postPushFetchMaxAttempts: Number.parseInt(process.env.fetch_max_attempts || '0', 10) } };",
"console.log(JSON.stringify(payload));",
"NODE",
"cat /cache/HWLAB.last-flush.json",
"if [ \"$exit_code\" != \"0\" ]; then exit \"$exit_code\"; fi",
"",
].join("\n");
}
function validateBenchmarkProfileName(value: string, path: string): void {
if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 63) throw new Error(`${path} must be a DNS-label style benchmark profile`);
}
@@ -3001,10 +2721,6 @@ function compactCommandResult(result: CommandResult): Record<string, unknown> {
};
}
function shQuote(value: string): string {
return `'${value.replace(/'/gu, `'"'"'`)}'`;
}
function validateHttpsUrl(value: string, path: string): void {
let parsed: URL;
try {
@@ -0,0 +1,65 @@
#!/bin/sh
set -eu
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
__GIT_MIRROR_PROXY_PRELUDE__
test -d "$repo/objects"
git --git-dir="$repo" remote set-url origin "$remote" || git --git-dir="$repo" remote add origin "$remote"
local_gitops=$(git --git-dir="$repo" rev-parse --verify "refs/heads/${gitops_branch}^{commit}" 2>/dev/null || true)
push_status=skipped
push_exit=0
fetch_status=skipped
fetch_exit=0
fetch_attempt=0
fetch_max_attempts=5
if [ -n "$local_gitops" ]; then
set +e
timeout 240 git --git-dir="$repo" -c remote.origin.mirror=false push origin "refs/heads/${gitops_branch}:refs/heads/${gitops_branch}"
push_exit=$?
set -e
if [ "$push_exit" = "0" ]; then
push_status=succeeded
fetch_retry_delay=1
while [ "$fetch_attempt" -lt "$fetch_max_attempts" ]; do
fetch_attempt=$((fetch_attempt + 1))
echo "git-mirror post-push fetch attempt ${fetch_attempt}/${fetch_max_attempts}" >&2
set +e
timeout 240 git --git-dir="$repo" fetch origin "+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}"
fetch_exit=$?
set -e
if [ "$fetch_exit" = "0" ]; then fetch_status=succeeded; break; fi
fetch_status=failed
if [ "$fetch_attempt" -lt "$fetch_max_attempts" ]; then
echo "git-mirror post-push fetch retry ${fetch_attempt}/${fetch_max_attempts} failed exit=${fetch_exit}; backoff=${fetch_retry_delay}s" >&2
sleep "$fetch_retry_delay"
if [ "$fetch_retry_delay" -lt 16 ]; then fetch_retry_delay=$((fetch_retry_delay * 2)); fi
fi
done
else
push_status=failed
fi
fi
github_gitops=$(git --git-dir="$repo" rev-parse --verify "refs/mirror-stage/heads/${gitops_branch}^{commit}" 2>/dev/null || true)
pending=false; if [ -n "$local_gitops" ] && { [ -z "$github_gitops" ] || [ "$local_gitops" != "$github_gitops" ]; }; then pending=true; fi
status=succeeded
partial_success=
degraded_reason=
exit_code=0
if [ "$push_status" = "failed" ]; then
status=failed
degraded_reason=git-mirror-push-failed
exit_code=$push_exit
elif [ "$push_status" = "succeeded" ] && [ "$fetch_status" = "failed" ]; then
status=partial-success
partial_success=push-succeeded-fetch-failed
degraded_reason=git-mirror-post-push-fetch-failed
exit_code=44
fi
export repository gitops_branch started_at local_gitops github_gitops pending push_status push_exit fetch_status fetch_exit fetch_attempt fetch_max_attempts status partial_success degraded_reason
summary_tmp=/tmp/HWLAB.last-flush.json.$$
node <<'NODE' > "$summary_tmp"
const payload = { event: 'git-mirror-flush', repo: process.env.repository, status: process.env.status || 'failed', partialSuccess: process.env.partial_success || null, degradedReason: process.env.degraded_reason || null, startedAt: process.env.started_at, flushedAt: new Date().toISOString(), gitopsBranch: process.env.gitops_branch, localGitops: process.env.local_gitops || null, githubGitops: process.env.github_gitops || null, pendingFlush: process.env.pending === 'true', stages: { push: process.env.push_status || null, pushExitCode: Number.parseInt(process.env.push_exit || '0', 10), postPushFetch: process.env.fetch_status || null, postPushFetchExitCode: Number.parseInt(process.env.fetch_exit || '0', 10), postPushFetchAttempts: Number.parseInt(process.env.fetch_attempt || '0', 10), postPushFetchMaxAttempts: Number.parseInt(process.env.fetch_max_attempts || '0', 10) } };
console.log(JSON.stringify(payload));
NODE
tee /cache/HWLAB.last-flush.json < "$summary_tmp"
rm -f "$summary_tmp"
if [ "$exit_code" != "0" ]; then exit "$exit_code"; fi
@@ -0,0 +1,48 @@
#!/usr/bin/env node
const net = require('node:net');
const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);
const proxyPort = Number.parseInt(proxyPortRaw || '', 10);
const targetPort = Number.parseInt(targetPortRaw || '', 10);
if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {
console.error('hwlab git-mirror proxy-connect: invalid ProxyCommand arguments');
process.exit(64);
}
let settled = false;
let tunnelEstablished = false;
function finish(code, message) {
if (settled) return;
settled = true;
if (message) console.error('hwlab git-mirror proxy-connect: ' + message);
process.exit(code);
}
const socket = net.createConnection({ host: proxyHost, port: proxyPort });
let buffer = Buffer.alloc(0);
socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });
socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\r\nHost: ' + targetHost + ':' + targetPort + '\r\nProxy-Connection: Keep-Alive\r\n\r\n'));
socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));
socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });
function onData(chunk) {
buffer = Buffer.concat([buffer, chunk]);
const headerEnd = buffer.indexOf('\r\n\r\n');
if (headerEnd === -1 && buffer.length < 8192) return;
if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }
const head = buffer.slice(0, headerEnd + 4).toString('latin1');
const statusLine = head.split('\r\n', 1)[0] || '';
const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);
if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {
const safeStatus = statusLine.replace(/[^\x20-\x7e]/g, '?').slice(0, 160);
socket.destroy();
finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);
return;
}
socket.off('data', onData);
socket.setTimeout(0);
tunnelEstablished = true;
const rest = buffer.slice(headerEnd + 4);
if (rest.length) process.stdout.write(rest);
process.stdin.on('error', () => {});
process.stdout.on('error', () => {});
process.stdin.pipe(socket);
socket.pipe(process.stdout);
}
socket.on('data', onData);
@@ -0,0 +1,56 @@
#!/bin/sh
set -eu
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
__GIT_MIRROR_PROXY_PRELUDE__
mkdir -p "$(dirname "$repo")"
if [ -d "$repo/objects" ] && [ -f "$repo/HEAD" ]; then
git --git-dir="$repo" remote set-url origin "$remote" || git --git-dir="$repo" remote add origin "$remote"
else
rm -rf "$repo"
git init --bare "$repo"
git --git-dir="$repo" remote add origin "$remote"
fi
git --git-dir="$repo" config uploadpack.allowReachableSHA1InWant true
git --git-dir="$repo" config uploadpack.allowAnySHA1InWant true
git --git-dir="$repo" config http.uploadpack true
git --git-dir="$repo" config http.receivepack true
timeout 240 git --git-dir="$repo" fetch origin "+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}"
source_sha=$(git --git-dir="$repo" rev-parse --verify "refs/mirror-stage/heads/${source_branch}^{commit}")
source_stage_ref="${source_stage_ref_prefix%/}/$source_sha"
git --git-dir="$repo" update-ref "$source_stage_ref" "$source_sha"
git --git-dir="$repo" update-ref "refs/heads/${source_branch}" "$source_sha"
discarded_stale_gitops=false
if timeout 240 git --git-dir="$repo" fetch origin "+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}"; then
github_gitops=$(git --git-dir="$repo" rev-parse --verify "refs/mirror-stage/heads/${gitops_branch}^{commit}" 2>/dev/null || true)
local_gitops=$(git --git-dir="$repo" rev-parse --verify "refs/heads/${gitops_branch}^{commit}" 2>/dev/null || true)
if [ -z "$local_gitops" ] && [ -n "$github_gitops" ]; then
git --git-dir="$repo" update-ref "refs/heads/${gitops_branch}" "$github_gitops"
elif [ -n "$local_gitops" ] && [ -n "$github_gitops" ] && [ "$local_gitops" != "$github_gitops" ] && git --git-dir="$repo" merge-base --is-ancestor "$local_gitops" "$github_gitops"; then
git --git-dir="$repo" update-ref "refs/heads/${gitops_branch}" "$github_gitops"
elif [ -n "$local_gitops" ] && [ -n "$github_gitops" ] && [ "$local_gitops" != "$github_gitops" ] && [ "${UNIDESK_GIT_MIRROR_DISCARD_STALE_GITOPS:-false}" = "true" ]; then
git --git-dir="$repo" update-ref "refs/heads/${gitops_branch}" "$github_gitops"
discarded_stale_gitops=true
printf '%s\n' "git-mirror sync: discarded stale local gitops ref local=${local_gitops} github=${github_gitops}" >&2
fi
fi
git --git-dir="$repo" update-server-info
export repository source_branch source_stage_ref gitops_branch started_at discarded_stale_gitops
summary_tmp=/tmp/HWLAB.last-sync.json.$$
node <<'NODE' > "$summary_tmp"
const { execFileSync } = require('node:child_process');
const repository = process.env.repository;
const sourceBranch = process.env.source_branch;
const gitopsBranch = process.env.gitops_branch;
const repoPath = `/cache/${repository}.git`;
function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }
const localSource = rev(`refs/heads/${sourceBranch}`);
const githubSource = rev(`refs/mirror-stage/heads/${sourceBranch}`);
const sourceStageRef = process.env.source_stage_ref;
const sourceSnapshot = sourceStageRef ? rev(sourceStageRef) : null;
const localGitops = rev(`refs/heads/${gitopsBranch}`);
const githubGitops = rev(`refs/mirror-stage/heads/${gitopsBranch}`);
const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));
console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), sourceAuthority: 'git-mirror-snapshot', localSource, githubSource, sourceStageRef, sourceSnapshot, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));
NODE
tee /cache/HWLAB.last-sync.json < "$summary_tmp"
rm -f "$summary_tmp"