fix: expose node git mirror flush partial success

scripts/src/hwlab-node.ts

@@ -495,6 +495,63 @@ function compactRuntimeCommand(result: CommandResult): Record<string, unknown> {
   };
 }
 
+function parseLastJsonLineObject(text: string): Record<string, unknown> {
+  for (const line of text.split(/\r?\n/u).reverse()) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) continue;
+    try {
+      return record(JSON.parse(trimmed) as unknown);
+    } catch {
+      // Keep scanning: command tails can mix kubectl status, git output, and JSON payloads.
+    }
+  }
+  return {};
+}
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+}
+
+function nodeRuntimeGitMirrorRefSources(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown> {
+  return {
+    localSource: `refs/heads/${mirror.sourceBranch}`,
+    githubSource: `refs/mirror-stage/heads/${mirror.sourceBranch}`,
+    localGitops: `refs/heads/${mirror.gitopsBranch}`,
+    githubGitops: `refs/mirror-stage/heads/${mirror.gitopsBranch}`,
+    githubFieldsAreMirrorStageCache: true,
+    cacheRefresh: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
+  };
+}
+
+function nodeRuntimeGitMirrorFlushPartialSuccess(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, mirror: NodeRuntimeGitMirrorTargetSpec, result: CommandResult): Record<string, unknown> | null {
+  if (scoped.action !== "flush") return null;
+  const text = `${result.stdout}\n${result.stderr}`;
+  const payload = parseLastJsonLineObject(text);
+  const partialSuccess = typeof payload.partialSuccess === "string" && payload.partialSuccess.length > 0
+    ? payload.partialSuccess
+    : null;
+  const payloadStatus = typeof payload.status === "string" && payload.status.length > 0 ? payload.status : null;
+  const branch = escapeRegExp(mirror.gitopsBranch);
+  const pushSucceededInGitOutput = new RegExp(`\\b${branch}\\s*->\\s*${branch}\\b`, "u").test(text);
+  const postPushFetchFailed = /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|fatal:.*fetch|fetch-pack|early EOF/iu.test(text);
+  if (partialSuccess !== "push-succeeded-fetch-failed" && !(result.exitCode !== 0 && pushSucceededInGitOutput && postPushFetchFailed)) {
+    return null;
+  }
+  return {
+    status: payloadStatus ?? "partial-success",
+    partialSuccess: "push-succeeded-fetch-failed",
+    degradedReason: "node-runtime-git-mirror-flush-post-push-fetch-failed",
+    retryable: true,
+    message: "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Refresh mirror-stage cache before deciding another flush is needed.",
+    payload: Object.keys(payload).length > 0 ? payload : null,
+    refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
+    next: {
+      sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
+      status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
+    },
+  };
+}
+
 function sshTcpPoolDiagnosticsFromCommand(spec: HwlabRuntimeLaneSpec, result: CommandResult): Record<string, unknown> | null {
   if (isCommandSuccess(result)) return null;
   const failureKind = classifySshTcpPoolFailure(`${result.stderr}\n${result.stdout}`);
@@ -1347,6 +1404,7 @@ function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeScopedDel
     gitopsBranch: mirror.gitopsBranch,
     resources,
     summary,
+    refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
     result: compactRuntimeCommand(result),
     degradedReason: ok ? undefined : "node-runtime-git-mirror-not-ready",
     valuesPrinted: false,
@@ -1392,6 +1450,7 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
   ].join("\n");
   const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
   const status = scoped.dryRun || !isCommandSuccess(result) ? undefined : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
+  const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
   return {
     ok: isCommandSuccess(result) && (status === undefined || status.ok === true),
     command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
@@ -1404,8 +1463,14 @@ function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelega
     manifest: scoped.dryRun ? manifest : undefined,
     result: compactRuntimeCommand(result),
     status,
-    degradedReason: isCommandSuccess(result) ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
-    next: scoped.dryRun
+    partialSuccess: partialSuccess?.partialSuccess ?? undefined,
+    recovery: partialSuccess ?? undefined,
+    degradedReason: partialSuccess !== null
+      ? "node-runtime-git-mirror-flush-post-push-fetch-failed"
+      : isCommandSuccess(result) ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
+    next: partialSuccess !== null
+      ? partialSuccess.next
+      : scoped.dryRun
       ? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` }
       : { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
   };