From 992e5da3b454ef7e588f4391c16997cd4deea3d2 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Jun 2026 17:34:50 +0000 Subject: [PATCH] fix(web-probe): retry sentinel source mirror fetch --- scripts/src/hwlab-node-web-sentinel-cicd.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 036bf20c..9f64a58b 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -1145,7 +1145,14 @@ function sentinelSourceMirrorSyncShell(state: SentinelCicdState, jobName: string "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", "git --git-dir=\"$repo\" config http.uploadpack true", "git --git-dir=\"$repo\" config http.receivepack true", - "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"", + "fetch_ok=0", + "for attempt in 1 2 3; do", + " if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/$source_branch:refs/mirror-stage/heads/$source_branch\"; then fetch_ok=1; break; fi", + " code=$?", + " printf '%s\\n' \"sentinel source-mirror fetch attempt ${attempt}/3 failed exit=${code}; retrying\" >&2", + " sleep $((attempt * 5))", + "done", + "test \"$fetch_ok\" = 1", "mirror_commit=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/$source_branch^{commit}\")", "test \"$mirror_commit\" = \"$source_commit\"", "git --git-dir=\"$repo\" update-ref \"refs/heads/$source_branch\" \"$mirror_commit\"", @@ -1177,7 +1184,7 @@ function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): strin "export no_proxy='*'", "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", "#!/bin/sh", - "exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 \"$@\"", + "exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 \"$@\"", "SH_PROXY", "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh", @@ -1220,7 +1227,7 @@ function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): strin "}", "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", "let buffer = Buffer.alloc(0);", - "socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });", + "socket.setTimeout(30000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });", "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", "socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));", "socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });", @@ -1253,7 +1260,7 @@ function sentinelSourceMirrorSshSetupShellLines(state: SentinelCicdState): strin "chmod 0700 /tmp/sentinel-github-proxy-connect.cjs", "cat > /tmp/sentinel-git-ssh-proxy.sh <<'SH_PROXY'", "#!/bin/sh", - `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shellQuote(proxyCommand)} "$@"`, + `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=30 -o ConnectionAttempts=2 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o ${shellQuote(proxyCommand)} "$@"`, "SH_PROXY", "chmod 0700 /tmp/sentinel-git-ssh-proxy.sh", "export GIT_SSH=/tmp/sentinel-git-ssh-proxy.sh",