From ea04a3810aa0e9e0390a281492f5b7e958c1a903 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Sun, 21 Jun 2026 10:40:25 +0800 Subject: [PATCH] fix(hwlab): stop node git mirror fallback (#555) Remove automatic host-workspace fallback and partial-success recovery from node-scoped git-mirror sync/flush. Retryable upstream GitHub SSH/fetch failures now stop with retryableFailure metadata and explicit retry/status next actions.\n\nRefs #554. Co-authored-by: Codex --- scripts/src/hwlab-node-impl.ts | 148 +++++++-------------------------- 1 file changed, 29 insertions(+), 119 deletions(-) diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 05b9c554..4c2e6256 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -2882,27 +2882,13 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType 0 - && !nodeRuntimeGitMirrorNeedsFlush(partialSuccessRecoveryStatus); - const actionSucceeded = isCommandSuccess(result) || fallback?.ok === true || partialSuccessRecovered; + const actionSucceeded = isCommandSuccess(result); const status = scoped.dryRun || !actionSucceeded ? undefined - : partialSuccessRecovered - ? partialSuccessRecovery?.status - : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); + : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); return { ok: actionSucceeded && (status === undefined || status.ok === true), command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, @@ -2914,119 +2900,43 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType, + mirror: NodeRuntimeGitMirrorTargetSpec, + result: CommandResult, + partialSuccess: Record | null, +): Record | null { const text = `${result.stdout}\n${result.stderr}`; - return /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com/iu.test(text); -} - -function nodeRuntimeGitMirrorSyncFallback(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec, timeoutSeconds: number): Record { - const writeUrl = nodeRuntimeGitMirrorHostWriteUrl(spec, mirror); - if (writeUrl === null) { - return { - ok: false, - mode: "host-workspace-fallback", - degradedReason: "node-runtime-git-mirror-write-service-unresolved", - next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` }, - }; - } - const script = [ - "set -eu", - `repository=${shellQuote(mirror.sourceRepository)}`, - `source_branch=${shellQuote(mirror.sourceBranch)}`, - `gitops_branch=${shellQuote(mirror.gitopsBranch)}`, - `mirror_write=${shellQuote(writeUrl)}`, - "github_remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", - "tmp_ns=\"refs/unidesk/git-mirror-sync/${source_branch}/$(date +%s)-$$\"", - "source_tmp=\"$tmp_ns/source\"", - "gitops_tmp=\"$tmp_ns/gitops\"", - "cleanup() { git update-ref -d \"$source_tmp\" >/dev/null 2>&1 || true; git update-ref -d \"$gitops_tmp\" >/dev/null 2>&1 || true; }", - "trap cleanup EXIT", - "export GIT_SSH_COMMAND='ssh -o HostKeyAlias=ssh.github.com -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", - "git fetch --no-tags \"$github_remote\" \"+refs/heads/${source_branch}:${source_tmp}\"", - "source_sha=$(git rev-parse --verify \"${source_tmp}^{commit}\")", - "git push \"$mirror_write\" \"$source_sha:refs/heads/${source_branch}\" \"$source_sha:refs/mirror-stage/heads/${source_branch}\"", - "gitops_sha=", - "if git fetch --no-tags \"$github_remote\" \"+refs/heads/${gitops_branch}:${gitops_tmp}\"; then", - " gitops_sha=$(git rev-parse --verify \"${gitops_tmp}^{commit}\")", - " git push \"$mirror_write\" \"$gitops_sha:refs/mirror-stage/heads/${gitops_branch}\"", - "fi", - "printf '{\"ok\":true,\"mode\":\"host-workspace-fallback\",\"sourceCommit\":\"%s\",\"gitopsCommit\":\"%s\",\"mirrorWrite\":\"%s\"}\\n' \"$source_sha\" \"$gitops_sha\" \"$mirror_write\"", - ].join("\n"); - const result = runTransWorkspaceStdinScript(spec.nodeId, spec.workspace, script, Math.max(60, Math.min(timeoutSeconds, 300))); + const retryable = partialSuccess !== null + || /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text); + if (!retryable) return null; return { - ok: isCommandSuccess(result), - mode: "host-workspace-fallback", - reason: "k3s git-mirror sync job could not reach GitHub SSH; fallback updates the node-local mirror from the target workspace through ssh.github.com:443", - mirrorWrite: writeUrl, - result: compactRuntimeCommand(result), - degradedReason: isCommandSuccess(result) ? undefined : "node-runtime-git-mirror-host-fallback-failed", - valuesPrinted: false, - }; -} - -function nodeRuntimeGitMirrorFlushFallback(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec, timeoutSeconds: number): Record { - const writeUrl = nodeRuntimeGitMirrorHostWriteUrl(spec, mirror); - if (writeUrl === null) { - return { - ok: false, - mode: "host-workspace-fallback", - degradedReason: "node-runtime-git-mirror-write-service-unresolved", - next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` }, - }; - } - const script = [ - "set -eu", - `repository=${shellQuote(mirror.sourceRepository)}`, - `gitops_branch=${shellQuote(mirror.gitopsBranch)}`, - `mirror_write=${shellQuote(writeUrl)}`, - "github_remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", - "tmp_ns=\"refs/unidesk/git-mirror-flush/${gitops_branch}/$(date +%s)-$$\"", - "local_gitops_tmp=\"$tmp_ns/local-gitops\"", - "github_gitops_tmp=\"$tmp_ns/github-gitops\"", - "cleanup() { git update-ref -d \"$local_gitops_tmp\" >/dev/null 2>&1 || true; git update-ref -d \"$github_gitops_tmp\" >/dev/null 2>&1 || true; }", - "trap cleanup EXIT", - "export GIT_SSH_COMMAND='ssh -o HostKeyAlias=ssh.github.com -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'", - "git fetch --no-tags \"$mirror_write\" \"+refs/heads/${gitops_branch}:${local_gitops_tmp}\"", - "local_gitops_sha=$(git rev-parse --verify \"${local_gitops_tmp}^{commit}\")", - "git push \"$github_remote\" \"$local_gitops_sha:refs/heads/${gitops_branch}\"", - "git fetch --no-tags \"$github_remote\" \"+refs/heads/${gitops_branch}:${github_gitops_tmp}\"", - "github_gitops_sha=$(git rev-parse --verify \"${github_gitops_tmp}^{commit}\")", - "if [ \"$github_gitops_sha\" != \"$local_gitops_sha\" ]; then", - " printf 'github gitops mismatch after fallback flush: local=%s github=%s\\n' \"$local_gitops_sha\" \"$github_gitops_sha\" >&2", - " exit 46", - "fi", - "git push \"$mirror_write\" \"$github_gitops_sha:refs/mirror-stage/heads/${gitops_branch}\"", - "printf '{\"ok\":true,\"mode\":\"host-workspace-fallback\",\"gitopsCommit\":\"%s\",\"mirrorWrite\":\"%s\"}\\n' \"$github_gitops_sha\" \"$mirror_write\"", - ].join("\n"); - const result = runTransWorkspaceStdinScript(spec.nodeId, spec.workspace, script, Math.max(60, Math.min(timeoutSeconds, 300))); - return { - ok: isCommandSuccess(result), - mode: "host-workspace-fallback", - reason: "k3s git-mirror flush job could not reach GitHub SSH; fallback pushes GitOps from the target workspace through ssh.github.com:443 and refreshes mirror-stage cache", - mirrorWrite: writeUrl, - result: compactRuntimeCommand(result), - degradedReason: isCommandSuccess(result) ? undefined : "node-runtime-git-mirror-host-fallback-failed", + retryable: true, + retryMaxAttempts: 5, + retryLabel: "1/5", + reason: partialSuccess !== null + ? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback." + : "Git mirror job hit a retryable upstream GitHub SSH/fetch failure. Standard git-mirror stops without host workspace fallback.", + refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), + next: { + retry: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`, + status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, + }, valuesPrinted: false, }; }