fix(hwlab): stop node git mirror fallback (#555)
Remove automatic host-workspace fallback and partial-success recovery from node-scoped git-mirror sync/flush. Retryable upstream GitHub SSH/fetch failures now stop with retryableFailure metadata and explicit retry/status next actions.\n\nRefs #554. Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
+29
-119
@@ -2882,27 +2882,13 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
|
||||
].join("\n");
|
||||
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
||||
const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
|
||||
const syncFallback = scoped.action === "sync" && !scoped.dryRun && !isCommandSuccess(result) && nodeRuntimeGitMirrorSyncFallbackAllowed(result)
|
||||
? nodeRuntimeGitMirrorSyncFallback(scoped.spec, mirror, scoped.timeoutSeconds)
|
||||
const retryableFailure = !isCommandSuccess(result)
|
||||
? nodeRuntimeGitMirrorRetryableFailure(scoped, mirror, result, partialSuccess)
|
||||
: null;
|
||||
const flushFallback = scoped.action === "flush" && !scoped.dryRun && !isCommandSuccess(result) && nodeRuntimeGitMirrorSyncFallbackAllowed(result)
|
||||
? nodeRuntimeGitMirrorFlushFallback(scoped.spec, mirror, scoped.timeoutSeconds)
|
||||
: null;
|
||||
const fallback = syncFallback ?? flushFallback;
|
||||
const partialSuccessRecovery = partialSuccess !== null && !scoped.dryRun
|
||||
? nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true })
|
||||
: null;
|
||||
const partialSuccessRecoveryStatus = record(partialSuccessRecovery?.status);
|
||||
const partialSuccessRecovered = partialSuccess !== null
|
||||
&& partialSuccessRecovery?.ok === true
|
||||
&& Object.keys(partialSuccessRecoveryStatus).length > 0
|
||||
&& !nodeRuntimeGitMirrorNeedsFlush(partialSuccessRecoveryStatus);
|
||||
const actionSucceeded = isCommandSuccess(result) || fallback?.ok === true || partialSuccessRecovered;
|
||||
const actionSucceeded = isCommandSuccess(result);
|
||||
const status = scoped.dryRun || !actionSucceeded
|
||||
? undefined
|
||||
: partialSuccessRecovered
|
||||
? partialSuccessRecovery?.status
|
||||
: nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
||||
: nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
||||
return {
|
||||
ok: actionSucceeded && (status === undefined || status.ok === true),
|
||||
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
@@ -2914,119 +2900,43 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
|
||||
jobName,
|
||||
manifest: scoped.dryRun ? manifest : undefined,
|
||||
result: compactRuntimeCommand(result),
|
||||
fallback: fallback ?? undefined,
|
||||
status,
|
||||
retryableFailure: retryableFailure ?? undefined,
|
||||
partialSuccess: partialSuccess?.partialSuccess ?? undefined,
|
||||
partialSuccessRecovered: partialSuccessRecovered ? true : undefined,
|
||||
recovery: partialSuccess ?? undefined,
|
||||
postPushRecovery: partialSuccessRecovery === null ? undefined : compactNodeRuntimeGitMirrorRun(partialSuccessRecovery),
|
||||
degradedReason: partialSuccess !== null
|
||||
? partialSuccessRecovered
|
||||
? undefined
|
||||
: "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
||||
: actionSucceeded ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : fallback === null ? `node-runtime-git-mirror-${scoped.action}-failed` : `node-runtime-git-mirror-${scoped.action}-fallback-failed`,
|
||||
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
||||
: actionSucceeded ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
|
||||
next: partialSuccess !== null
|
||||
? partialSuccessRecovered
|
||||
? { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` }
|
||||
: partialSuccess.next
|
||||
? partialSuccess.next
|
||||
: scoped.dryRun
|
||||
? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` }
|
||||
: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
||||
: retryableFailure?.next ?? { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeGitMirrorSyncFallbackAllowed(result: CommandResult): boolean {
|
||||
function nodeRuntimeGitMirrorRetryableFailure(
|
||||
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
||||
mirror: NodeRuntimeGitMirrorTargetSpec,
|
||||
result: CommandResult,
|
||||
partialSuccess: Record<string, unknown> | null,
|
||||
): Record<string, unknown> | null {
|
||||
const text = `${result.stdout}\n${result.stderr}`;
|
||||
return /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com/iu.test(text);
|
||||
}
|
||||
|
||||
function nodeRuntimeGitMirrorSyncFallback(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec, timeoutSeconds: number): Record<string, unknown> {
|
||||
const writeUrl = nodeRuntimeGitMirrorHostWriteUrl(spec, mirror);
|
||||
if (writeUrl === null) {
|
||||
return {
|
||||
ok: false,
|
||||
mode: "host-workspace-fallback",
|
||||
degradedReason: "node-runtime-git-mirror-write-service-unresolved",
|
||||
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` },
|
||||
};
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
`repository=${shellQuote(mirror.sourceRepository)}`,
|
||||
`source_branch=${shellQuote(mirror.sourceBranch)}`,
|
||||
`gitops_branch=${shellQuote(mirror.gitopsBranch)}`,
|
||||
`mirror_write=${shellQuote(writeUrl)}`,
|
||||
"github_remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
|
||||
"tmp_ns=\"refs/unidesk/git-mirror-sync/${source_branch}/$(date +%s)-$$\"",
|
||||
"source_tmp=\"$tmp_ns/source\"",
|
||||
"gitops_tmp=\"$tmp_ns/gitops\"",
|
||||
"cleanup() { git update-ref -d \"$source_tmp\" >/dev/null 2>&1 || true; git update-ref -d \"$gitops_tmp\" >/dev/null 2>&1 || true; }",
|
||||
"trap cleanup EXIT",
|
||||
"export GIT_SSH_COMMAND='ssh -o HostKeyAlias=ssh.github.com -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
||||
"git fetch --no-tags \"$github_remote\" \"+refs/heads/${source_branch}:${source_tmp}\"",
|
||||
"source_sha=$(git rev-parse --verify \"${source_tmp}^{commit}\")",
|
||||
"git push \"$mirror_write\" \"$source_sha:refs/heads/${source_branch}\" \"$source_sha:refs/mirror-stage/heads/${source_branch}\"",
|
||||
"gitops_sha=",
|
||||
"if git fetch --no-tags \"$github_remote\" \"+refs/heads/${gitops_branch}:${gitops_tmp}\"; then",
|
||||
" gitops_sha=$(git rev-parse --verify \"${gitops_tmp}^{commit}\")",
|
||||
" git push \"$mirror_write\" \"$gitops_sha:refs/mirror-stage/heads/${gitops_branch}\"",
|
||||
"fi",
|
||||
"printf '{\"ok\":true,\"mode\":\"host-workspace-fallback\",\"sourceCommit\":\"%s\",\"gitopsCommit\":\"%s\",\"mirrorWrite\":\"%s\"}\\n' \"$source_sha\" \"$gitops_sha\" \"$mirror_write\"",
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(spec.nodeId, spec.workspace, script, Math.max(60, Math.min(timeoutSeconds, 300)));
|
||||
const retryable = partialSuccess !== null
|
||||
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
|
||||
if (!retryable) return null;
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
mode: "host-workspace-fallback",
|
||||
reason: "k3s git-mirror sync job could not reach GitHub SSH; fallback updates the node-local mirror from the target workspace through ssh.github.com:443",
|
||||
mirrorWrite: writeUrl,
|
||||
result: compactRuntimeCommand(result),
|
||||
degradedReason: isCommandSuccess(result) ? undefined : "node-runtime-git-mirror-host-fallback-failed",
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeGitMirrorFlushFallback(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec, timeoutSeconds: number): Record<string, unknown> {
|
||||
const writeUrl = nodeRuntimeGitMirrorHostWriteUrl(spec, mirror);
|
||||
if (writeUrl === null) {
|
||||
return {
|
||||
ok: false,
|
||||
mode: "host-workspace-fallback",
|
||||
degradedReason: "node-runtime-git-mirror-write-service-unresolved",
|
||||
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` },
|
||||
};
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
`repository=${shellQuote(mirror.sourceRepository)}`,
|
||||
`gitops_branch=${shellQuote(mirror.gitopsBranch)}`,
|
||||
`mirror_write=${shellQuote(writeUrl)}`,
|
||||
"github_remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
|
||||
"tmp_ns=\"refs/unidesk/git-mirror-flush/${gitops_branch}/$(date +%s)-$$\"",
|
||||
"local_gitops_tmp=\"$tmp_ns/local-gitops\"",
|
||||
"github_gitops_tmp=\"$tmp_ns/github-gitops\"",
|
||||
"cleanup() { git update-ref -d \"$local_gitops_tmp\" >/dev/null 2>&1 || true; git update-ref -d \"$github_gitops_tmp\" >/dev/null 2>&1 || true; }",
|
||||
"trap cleanup EXIT",
|
||||
"export GIT_SSH_COMMAND='ssh -o HostKeyAlias=ssh.github.com -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
||||
"git fetch --no-tags \"$mirror_write\" \"+refs/heads/${gitops_branch}:${local_gitops_tmp}\"",
|
||||
"local_gitops_sha=$(git rev-parse --verify \"${local_gitops_tmp}^{commit}\")",
|
||||
"git push \"$github_remote\" \"$local_gitops_sha:refs/heads/${gitops_branch}\"",
|
||||
"git fetch --no-tags \"$github_remote\" \"+refs/heads/${gitops_branch}:${github_gitops_tmp}\"",
|
||||
"github_gitops_sha=$(git rev-parse --verify \"${github_gitops_tmp}^{commit}\")",
|
||||
"if [ \"$github_gitops_sha\" != \"$local_gitops_sha\" ]; then",
|
||||
" printf 'github gitops mismatch after fallback flush: local=%s github=%s\\n' \"$local_gitops_sha\" \"$github_gitops_sha\" >&2",
|
||||
" exit 46",
|
||||
"fi",
|
||||
"git push \"$mirror_write\" \"$github_gitops_sha:refs/mirror-stage/heads/${gitops_branch}\"",
|
||||
"printf '{\"ok\":true,\"mode\":\"host-workspace-fallback\",\"gitopsCommit\":\"%s\",\"mirrorWrite\":\"%s\"}\\n' \"$github_gitops_sha\" \"$mirror_write\"",
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(spec.nodeId, spec.workspace, script, Math.max(60, Math.min(timeoutSeconds, 300)));
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
mode: "host-workspace-fallback",
|
||||
reason: "k3s git-mirror flush job could not reach GitHub SSH; fallback pushes GitOps from the target workspace through ssh.github.com:443 and refreshes mirror-stage cache",
|
||||
mirrorWrite: writeUrl,
|
||||
result: compactRuntimeCommand(result),
|
||||
degradedReason: isCommandSuccess(result) ? undefined : "node-runtime-git-mirror-host-fallback-failed",
|
||||
retryable: true,
|
||||
retryMaxAttempts: 5,
|
||||
retryLabel: "1/5",
|
||||
reason: partialSuccess !== null
|
||||
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
|
||||
: "Git mirror job hit a retryable upstream GitHub SSH/fetch failure. Standard git-mirror stops without host workspace fallback.",
|
||||
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
||||
next: {
|
||||
retry: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
||||
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user