From 46c316b6b1a3feb153542e87394691c2918462b1 Mon Sep 17 00:00:00 2001
From: Codex <codex@noreply.local>
Date: Wed, 17 Jun 2026 16:22:14 +0000
Subject: [PATCH] fix: fallback hwlab git mirror source sync

---
 scripts/src/hwlab-node.ts | 71 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 67 insertions(+), 4 deletions(-)

diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts
index 24d17ae1..e61a4599 100644
--- a/scripts/src/hwlab-node.ts
+++ b/scripts/src/hwlab-node.ts
@@ -2266,25 +2266,30 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
       ].join("\n"),
   ].join("\n");
   const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
-  const status = scoped.dryRun || !isCommandSuccess(result) ? undefined : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
+  const fallback = scoped.action === "sync" && !scoped.dryRun && !isCommandSuccess(result) && nodeRuntimeGitMirrorSyncFallbackAllowed(result)
+    ? nodeRuntimeGitMirrorSyncFallback(scoped.spec, mirror, scoped.timeoutSeconds)
+    : null;
+  const actionSucceeded = isCommandSuccess(result) || fallback?.ok === true;
+  const status = scoped.dryRun || !actionSucceeded ? undefined : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
   const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
   return {
-    ok: isCommandSuccess(result) && (status === undefined || status.ok === true),
+    ok: actionSucceeded && (status === undefined || status.ok === true),
     command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
     node: scoped.node,
     lane: scoped.lane,
     mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`,
-    mutation: !scoped.dryRun && isCommandSuccess(result),
+    mutation: !scoped.dryRun && actionSucceeded,
     namespace: mirror.namespace,
     jobName,
     manifest: scoped.dryRun ? manifest : undefined,
     result: compactRuntimeCommand(result),
+    fallback: fallback ?? undefined,
     status,
     partialSuccess: partialSuccess?.partialSuccess ?? undefined,
     recovery: partialSuccess ?? undefined,
     degradedReason: partialSuccess !== null
       ? "node-runtime-git-mirror-flush-post-push-fetch-failed"
-      : isCommandSuccess(result) ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
+      : actionSucceeded ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : fallback === null ? `node-runtime-git-mirror-${scoped.action}-failed` : "node-runtime-git-mirror-sync-fallback-failed",
     next: partialSuccess !== null
       ? partialSuccess.next
       : scoped.dryRun
@@ -2293,6 +2298,63 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
   };
 }
 
+function nodeRuntimeGitMirrorSyncFallbackAllowed(result: CommandResult): boolean {
+  const text = `${result.stdout}\n${result.stderr}`;
+  return /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com/iu.test(text);
+}
+
+function nodeRuntimeGitMirrorSyncFallback(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec, timeoutSeconds: number): Record<string, unknown> {
+  const writeUrl = nodeRuntimeGitMirrorHostWriteUrl(spec, mirror);
+  if (writeUrl === null) {
+    return {
+      ok: false,
+      mode: "host-workspace-fallback",
+      degradedReason: "node-runtime-git-mirror-write-service-unresolved",
+      next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` },
+    };
+  }
+  const script = [
+    "set -eu",
+    `repository=${shellQuote(mirror.sourceRepository)}`,
+    `source_branch=${shellQuote(mirror.sourceBranch)}`,
+    `gitops_branch=${shellQuote(mirror.gitopsBranch)}`,
+    `mirror_write=${shellQuote(writeUrl)}`,
+    "github_remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
+    "tmp_ns=\"refs/unidesk/git-mirror-sync/${source_branch}/$(date +%s)-$$\"",
+    "source_tmp=\"$tmp_ns/source\"",
+    "gitops_tmp=\"$tmp_ns/gitops\"",
+    "cleanup() { git update-ref -d \"$source_tmp\" >/dev/null 2>&1 || true; git update-ref -d \"$gitops_tmp\" >/dev/null 2>&1 || true; }",
+    "trap cleanup EXIT",
+    "export GIT_SSH_COMMAND='ssh -o HostKeyAlias=ssh.github.com -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -o ServerAliveInterval=5 -o ServerAliveCountMax=1'",
+    "git fetch --no-tags \"$github_remote\" \"+refs/heads/${source_branch}:${source_tmp}\"",
+    "source_sha=$(git rev-parse --verify \"${source_tmp}^{commit}\")",
+    "git push \"$mirror_write\" \"$source_sha:refs/heads/${source_branch}\" \"$source_sha:refs/mirror-stage/heads/${source_branch}\"",
+    "gitops_sha=",
+    "if git fetch --no-tags \"$github_remote\" \"+refs/heads/${gitops_branch}:${gitops_tmp}\"; then",
+    "  gitops_sha=$(git rev-parse --verify \"${gitops_tmp}^{commit}\")",
+    "  git push \"$mirror_write\" \"$gitops_sha:refs/mirror-stage/heads/${gitops_branch}\"",
+    "fi",
+    "printf '{\"ok\":true,\"mode\":\"host-workspace-fallback\",\"sourceCommit\":\"%s\",\"gitopsCommit\":\"%s\",\"mirrorWrite\":\"%s\"}\\n' \"$source_sha\" \"$gitops_sha\" \"$mirror_write\"",
+  ].join("\n");
+  const result = runTransWorkspaceStdinScript(spec.nodeId, spec.workspace, script, Math.max(60, Math.min(timeoutSeconds, 300)));
+  return {
+    ok: isCommandSuccess(result),
+    mode: "host-workspace-fallback",
+    reason: "k3s git-mirror sync job could not reach GitHub SSH; fallback updates the node-local mirror from the target workspace through ssh.github.com:443",
+    mirrorWrite: writeUrl,
+    result: compactRuntimeCommand(result),
+    degradedReason: isCommandSuccess(result) ? undefined : "node-runtime-git-mirror-host-fallback-failed",
+    valuesPrinted: false,
+  };
+}
+
+function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec): string | null {
+  const result = runNodeK3sArgs(spec, ["kubectl", "-n", mirror.namespace, "get", "service", mirror.serviceWriteName, "-o", "jsonpath={.spec.clusterIP}{\":\"}{.spec.ports[0].port}"], 60);
+  const value = result.stdout.trim();
+  if (result.exitCode !== 0 || value.length === 0 || value.startsWith(":")) return null;
+  return `http://${value}/${mirror.sourceRepository}.git`;
+}
+
 function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
   const full = nodeScopedFullOutput(scoped);
   const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
@@ -2435,6 +2497,7 @@ function compactNodeRuntimeGitMirrorRun(result: Record<string, unknown>): Record
     mutation: result.mutation === true,
     jobName: result.jobName ?? null,
     partialSuccess: result.partialSuccess ?? null,
+    fallback: result.fallback ?? null,
     degradedReason: result.degradedReason ?? null,
     statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null,
     next: result.next ?? null,