Merge pull request #418 from pikasTech/fix/agentrun-gitops-publish-async

修复 AgentRun GitOps publish 长连接部署卡点

scripts/src/agentrun.ts

@@ -2618,9 +2618,35 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen
   }
   const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" });
   const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image });
-  const gitops = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishScript(spec, renderedFiles)]);
+  progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
-  const gitopsPayload = captureJsonPayload(gitops);
+    node: spec.nodeId,
-  if (gitops.exitCode !== 0 || gitopsPayload.ok === false) {
+    lane: spec.lane,
+    sourceCommit,
+    status: "submitting",
+  });
+  const gitopsSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishSubmitScript(spec, renderedFiles)]);
+  const gitopsSubmitPayload = captureJsonPayload(gitopsSubmit);
+  if (gitopsSubmit.exitCode !== 0 || gitopsSubmitPayload.ok === false) {
+    return {
+      ok: false,
+      command: "agentrun control-plane trigger-current",
+      mode: waited ? "confirmed-waited" : "confirmed-trigger",
+      configPath,
+      target: agentRunLaneSummary(spec),
+      phase: "gitops-publish-submit",
+      sourceCommit,
+      image,
+      degradedReason: "yaml-lane-gitops-publish-submit-failed",
+      sourceBootstrap: bootstrapPayload,
+      imageBuild: buildPayload,
+      result: gitopsSubmitPayload,
+      capture: compactCapture(gitopsSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
+      valuesPrinted: false,
+    };
+  }
+  const gitops = await waitForYamlLaneGitopsPublish(config, spec, sourceCommit, stringOrNull(gitopsSubmitPayload.jobId));
+  const gitopsPayload = gitops.payload;
+  if (gitops.ok !== true || gitopsPayload.ok === false) {
     return {
       ok: false,
       command: "agentrun control-plane trigger-current",
@@ -2633,8 +2659,9 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen
       degradedReason: "yaml-lane-gitops-publish-failed",
       sourceBootstrap: bootstrapPayload,
       imageBuild: buildPayload,
+      gitopsPublishSubmit: gitopsSubmitPayload,
       result: gitopsPayload,
-      capture: compactCapture(gitops, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
+      gitopsPublishStatus: gitops,
       valuesPrinted: false,
     };
   }
@@ -2671,6 +2698,7 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen
     sourceBootstrap: bootstrapPayload,
     imageBuildSubmit: buildSubmitPayload,
     imageBuild: buildPayload,
+    gitopsPublishSubmit: gitopsSubmitPayload,
     renderedFiles: {
       count: renderedFiles.length,
       digest: renderedFilesDigest(renderedFiles),
@@ -3262,13 +3290,15 @@ function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string):
   ].join("\n");
 }
 
-function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string {
+function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string {
+  const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
   const filesB64 = Buffer.from(JSON.stringify(files.map((file) => ({
     path: file.path,
     contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
   }))), "utf8").toString("base64");
   return [
     "set -eu",
+    `state_dir=${shQuote(stateDir)}`,
     `workspace=${shQuote(spec.source.workspace)}`,
     `remote=${shQuote(spec.source.remote)}`,
     `source_branch=${shQuote(spec.source.branch)}`,
@@ -3276,17 +3306,34 @@ function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { p
     `gitops_root=${shQuote(spec.deployment.gitopsRoot)}`,
     `artifact_catalog=${shQuote(spec.deployment.artifactCatalogPath)}`,
     `files_b64=${shQuote(filesB64)}`,
-    "cd \"$workspace\"",
+    "mkdir -p \"$state_dir\"",
-    "git fetch origin \"$gitops_branch\" || true",
+    "job_id=\"gitops-publish-$(date +%s)-$$\"",
-    "if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then",
+    "status_file=\"$state_dir/$job_id.json\"",
-    "  git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"",
+    "stdout_file=\"$state_dir/$job_id.stdout.log\"",
-    "else",
+    "stderr_file=\"$state_dir/$job_id.stderr.log\"",
-    "  git checkout --orphan \"$gitops_branch\"",
+    "cat > \"$status_file\" <<EOF",
-    "  git rm -rf . >/dev/null 2>&1 || true",
+    "{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"workspace\":\"$workspace\",\"gitopsBranch\":\"$gitops_branch\",\"valuesPrinted\":false}",
-    "fi",
+    "EOF",
-    "git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true",
+    "(",
-    "rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json",
+    "  set -eu",
-    "FILES_B64=\"$files_b64\" node <<'NODE'",
+    "  write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 80 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" GITOPS_BRANCH=\"$gitops_branch\" node <<'NODE' > \"$status_file\"",
+    "const code = Number(process.env.CODE || 1);",
+    "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, gitopsBranch: process.env.GITOPS_BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
+    "NODE",
+    "  fi; exit \"$code\"; }",
+    "  trap write_failed_status EXIT",
+    "  cd \"$workspace\"",
+    "  git remote set-url origin \"$remote\" || git remote add origin \"$remote\"",
+    "  git fetch origin \"$gitops_branch\" || true",
+    "  if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then",
+    "    git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"",
+    "  else",
+    "    git checkout --orphan \"$gitops_branch\"",
+    "    git rm -rf . >/dev/null 2>&1 || true",
+    "  fi",
+    "  git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true",
+    "  rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json",
+    "  FILES_B64=\"$files_b64\" node <<'NODE'",
     "const fs = require('node:fs');",
     "const path = require('node:path');",
     "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));",
@@ -3297,14 +3344,57 @@ function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { p
     "  fs.writeFileSync(target, Buffer.from(file.contentBase64, 'base64'));",
     "}",
     "NODE",
-    "git add source.json \"$artifact_catalog\" \"$gitops_root\"",
+    "  git add source.json \"$artifact_catalog\" \"$gitops_root\"",
-    "if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi",
+    "  if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi",
-    "git push -u origin \"$gitops_branch\"",
+    "  git push -u origin \"$gitops_branch\"",
-    "gitops_commit=$(git rev-parse HEAD)",
+    "  gitops_commit=$(git rev-parse HEAD)",
-    "git checkout \"$source_branch\" >/dev/null 2>&1 || true",
+    "  git checkout \"$source_branch\" >/dev/null 2>&1 || true",
-    "CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" node <<'NODE'",
+    "  CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" node <<'NODE' > \"$status_file\"",
-    "console.log(JSON.stringify({ ok: true, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));",
+    "console.log(JSON.stringify({ ok: true, status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));",
     "NODE",
+    "  trap - EXIT",
+    ") >\"$stdout_file\" 2>\"$stderr_file\" &",
+    "pid=$!",
+    "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
+    "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
+    "NODE",
+  ].join("\n");
+}
+
+async function waitForYamlLaneGitopsPublish(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
+  if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "gitops-publish-job-id-missing", valuesPrinted: false } };
+  const startedAt = Date.now();
+  const timeoutMs = 300_000;
+  let lastPayload: Record<string, unknown> = {};
+  let polls = 0;
+  while (Date.now() - startedAt < timeoutMs) {
+    polls += 1;
+    const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishStatusScript(spec, jobId)]);
+    const payload = captureJsonPayload(probe);
+    lastPayload = payload;
+    progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
+      node: spec.nodeId,
+      lane: spec.lane,
+      sourceCommit,
+      jobId,
+      polls,
+      status: stringOrNull(payload.status) ?? "unknown",
+      gitopsCommit: stringOrNull(payload.gitopsCommit),
+      elapsedMs: Date.now() - startedAt,
+    });
+    if (payload.ok === true && stringOrNull(payload.gitopsCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
+    if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
+    await sleep(5_000);
+  }
+  return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
+}
+
+function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
+  const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
+  return [
+    "set +e",
+    `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
+    "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
   ].join("\n");
 }