fix: fallback hwlab git mirror source sync
This commit is contained in:
Codex
@@ -2266,25 +2266,30 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
|
||||
].join("\n"),
|
||||
].join("\n");
|
||||
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
||||
const status = scoped.dryRun || !isCommandSuccess(result) ? undefined : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
||||
const fallback = scoped.action === "sync" && !scoped.dryRun && !isCommandSuccess(result) && nodeRuntimeGitMirrorSyncFallbackAllowed(result)
|
||||
? nodeRuntimeGitMirrorSyncFallback(scoped.spec, mirror, scoped.timeoutSeconds)
|
||||
: null;
|
||||
const actionSucceeded = isCommandSuccess(result) || fallback?.ok === true;
|
||||
const status = scoped.dryRun || !actionSucceeded ? undefined : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
||||
const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
|
||||
return {
|
||||
ok: isCommandSuccess(result) && (status === undefined || status.ok === true),
|
||||
ok: actionSucceeded && (status === undefined || status.ok === true),
|
||||
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`,
|
||||
mutation: !scoped.dryRun && isCommandSuccess(result),
|
||||
mutation: !scoped.dryRun && actionSucceeded,
|
||||
namespace: mirror.namespace,
|
||||
jobName,
|
||||
manifest: scoped.dryRun ? manifest : undefined,
|
||||
result: compactRuntimeCommand(result),
|
||||
fallback: fallback ?? undefined,
|
||||
status,
|
||||
partialSuccess: partialSuccess?.partialSuccess ?? undefined,
|
||||
recovery: partialSuccess ?? undefined,
|
||||
degradedReason: partialSuccess !== null
|
||||
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
||||
: isCommandSuccess(result) ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
|
||||
: actionSucceeded ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : fallback === null ? `node-runtime-git-mirror-${scoped.action}-failed` : "node-runtime-git-mirror-sync-fallback-failed",
|
||||
next: partialSuccess !== null
|
||||
? partialSuccess.next
|
||||
: scoped.dryRun
|
||||
@@ -2293,6 +2298,63 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeGitMirrorSyncFallbackAllowed(result: CommandResult): boolean {
|
||||
const text = `${result.stdout}\n${result.stderr}`;
|
||||
return /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com/iu.test(text);
|
||||
}
|
||||
|
||||
function nodeRuntimeGitMirrorSyncFallback(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec, timeoutSeconds: number): Record<string, unknown> {
|
||||
const writeUrl = nodeRuntimeGitMirrorHostWriteUrl(spec, mirror);
|
||||
if (writeUrl === null) {
|
||||
return {
|
||||
ok: false,
|
||||
mode: "host-workspace-fallback",
|
||||
degradedReason: "node-runtime-git-mirror-write-service-unresolved",
|
||||
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` },
|
||||
};
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
`repository=${shellQuote(mirror.sourceRepository)}`,
|
||||
`source_branch=${shellQuote(mirror.sourceBranch)}`,
|
||||
`gitops_branch=${shellQuote(mirror.gitopsBranch)}`,
|
||||
`mirror_write=${shellQuote(writeUrl)}`,
|
||||
"github_remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
|
||||
"tmp_ns=\"refs/unidesk/git-mirror-sync/${source_branch}/$(date +%s)-$$\"",
|
||||
"source_tmp=\"$tmp_ns/source\"",
|
||||
"gitops_tmp=\"$tmp_ns/gitops\"",
|
||||
"cleanup() { git update-ref -d \"$source_tmp\" >/dev/null 2>&1 || true; git update-ref -d \"$gitops_tmp\" >/dev/null 2>&1 || true; }",
|
||||
"trap cleanup EXIT",
|
||||
"export GIT_SSH_COMMAND='ssh -o HostKeyAlias=ssh.github.com -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
|
||||
"git fetch --no-tags \"$github_remote\" \"+refs/heads/${source_branch}:${source_tmp}\"",
|
||||
"source_sha=$(git rev-parse --verify \"${source_tmp}^{commit}\")",
|
||||
"git push \"$mirror_write\" \"$source_sha:refs/heads/${source_branch}\" \"$source_sha:refs/mirror-stage/heads/${source_branch}\"",
|
||||
"gitops_sha=",
|
||||
"if git fetch --no-tags \"$github_remote\" \"+refs/heads/${gitops_branch}:${gitops_tmp}\"; then",
|
||||
" gitops_sha=$(git rev-parse --verify \"${gitops_tmp}^{commit}\")",
|
||||
" git push \"$mirror_write\" \"$gitops_sha:refs/mirror-stage/heads/${gitops_branch}\"",
|
||||
"fi",
|
||||
"printf '{\"ok\":true,\"mode\":\"host-workspace-fallback\",\"sourceCommit\":\"%s\",\"gitopsCommit\":\"%s\",\"mirrorWrite\":\"%s\"}\\n' \"$source_sha\" \"$gitops_sha\" \"$mirror_write\"",
|
||||
].join("\n");
|
||||
const result = runTransWorkspaceStdinScript(spec.nodeId, spec.workspace, script, Math.max(60, Math.min(timeoutSeconds, 300)));
|
||||
return {
|
||||
ok: isCommandSuccess(result),
|
||||
mode: "host-workspace-fallback",
|
||||
reason: "k3s git-mirror sync job could not reach GitHub SSH; fallback updates the node-local mirror from the target workspace through ssh.github.com:443",
|
||||
mirrorWrite: writeUrl,
|
||||
result: compactRuntimeCommand(result),
|
||||
degradedReason: isCommandSuccess(result) ? undefined : "node-runtime-git-mirror-host-fallback-failed",
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec): string | null {
|
||||
const result = runNodeK3sArgs(spec, ["kubectl", "-n", mirror.namespace, "get", "service", mirror.serviceWriteName, "-o", "jsonpath={.spec.clusterIP}{\":\"}{.spec.ports[0].port}"], 60);
|
||||
const value = result.stdout.trim();
|
||||
if (result.exitCode !== 0 || value.length === 0 || value.startsWith(":")) return null;
|
||||
return `http://${value}/${mirror.sourceRepository}.git`;
|
||||
}
|
||||
|
||||
function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
|
||||
const full = nodeScopedFullOutput(scoped);
|
||||
const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
||||
@@ -2435,6 +2497,7 @@ function compactNodeRuntimeGitMirrorRun(result: Record<string, unknown>): Record
|
||||
mutation: result.mutation === true,
|
||||
jobName: result.jobName ?? null,
|
||||
partialSuccess: result.partialSuccess ?? null,
|
||||
fallback: result.fallback ?? null,
|
||||
degradedReason: result.degradedReason ?? null,
|
||||
statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null,
|
||||
next: result.next ?? null,
|
||||
|
||||
Reference in New Issue
Block a user