diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index c41750dc..76171933 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -2618,9 +2618,35 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen } const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" }); const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image }); - const gitops = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishScript(spec, renderedFiles)]); - const gitopsPayload = captureJsonPayload(gitops); - if (gitops.exitCode !== 0 || gitopsPayload.ok === false) { + progressEvent("agentrun.yaml-lane.gitops-publish.progress", { + node: spec.nodeId, + lane: spec.lane, + sourceCommit, + status: "submitting", + }); + const gitopsSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishSubmitScript(spec, renderedFiles)]); + const gitopsSubmitPayload = captureJsonPayload(gitopsSubmit); + if (gitopsSubmit.exitCode !== 0 || gitopsSubmitPayload.ok === false) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "gitops-publish-submit", + sourceCommit, + image, + degradedReason: "yaml-lane-gitops-publish-submit-failed", + sourceBootstrap: bootstrapPayload, + imageBuild: buildPayload, + result: gitopsSubmitPayload, + capture: compactCapture(gitopsSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const gitops = await waitForYamlLaneGitopsPublish(config, spec, sourceCommit, stringOrNull(gitopsSubmitPayload.jobId)); + const gitopsPayload = gitops.payload; + if (gitops.ok !== true || gitopsPayload.ok === false) { return { ok: false, command: "agentrun control-plane trigger-current", @@ -2633,8 +2659,9 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen degradedReason: "yaml-lane-gitops-publish-failed", sourceBootstrap: bootstrapPayload, imageBuild: buildPayload, + gitopsPublishSubmit: gitopsSubmitPayload, result: gitopsPayload, - capture: compactCapture(gitops, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + gitopsPublishStatus: gitops, valuesPrinted: false, }; } @@ -2671,6 +2698,7 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen sourceBootstrap: bootstrapPayload, imageBuildSubmit: buildSubmitPayload, imageBuild: buildPayload, + gitopsPublishSubmit: gitopsSubmitPayload, renderedFiles: { count: renderedFiles.length, digest: renderedFilesDigest(renderedFiles), @@ -3262,13 +3290,15 @@ function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string): ].join("\n"); } -function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string { +function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string { + const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`; const filesB64 = Buffer.from(JSON.stringify(files.map((file) => ({ path: file.path, contentBase64: Buffer.from(file.content, "utf8").toString("base64"), }))), "utf8").toString("base64"); return [ "set -eu", + `state_dir=${shQuote(stateDir)}`, `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `source_branch=${shQuote(spec.source.branch)}`, @@ -3276,17 +3306,34 @@ function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { p `gitops_root=${shQuote(spec.deployment.gitopsRoot)}`, `artifact_catalog=${shQuote(spec.deployment.artifactCatalogPath)}`, `files_b64=${shQuote(filesB64)}`, - "cd \"$workspace\"", - "git fetch origin \"$gitops_branch\" || true", - "if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then", - " git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"", - "else", - " git checkout --orphan \"$gitops_branch\"", - " git rm -rf . >/dev/null 2>&1 || true", - "fi", - "git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true", - "rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json", - "FILES_B64=\"$files_b64\" node <<'NODE'", + "mkdir -p \"$state_dir\"", + "job_id=\"gitops-publish-$(date +%s)-$$\"", + "status_file=\"$state_dir/$job_id.json\"", + "stdout_file=\"$state_dir/$job_id.stdout.log\"", + "stderr_file=\"$state_dir/$job_id.stderr.log\"", + "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" GITOPS_BRANCH=\"$gitops_branch\" node <<'NODE' > \"$status_file\"", + "const code = Number(process.env.CODE || 1);", + "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, gitopsBranch: process.env.GITOPS_BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", + "NODE", + " fi; exit \"$code\"; }", + " trap write_failed_status EXIT", + " cd \"$workspace\"", + " git remote set-url origin \"$remote\" || git remote add origin \"$remote\"", + " git fetch origin \"$gitops_branch\" || true", + " if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then", + " git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"", + " else", + " git checkout --orphan \"$gitops_branch\"", + " git rm -rf . >/dev/null 2>&1 || true", + " fi", + " git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true", + " rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json", + " FILES_B64=\"$files_b64\" node <<'NODE'", "const fs = require('node:fs');", "const path = require('node:path');", "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));", @@ -3297,14 +3344,57 @@ function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { p " fs.writeFileSync(target, Buffer.from(file.contentBase64, 'base64'));", "}", "NODE", - "git add source.json \"$artifact_catalog\" \"$gitops_root\"", - "if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi", - "git push -u origin \"$gitops_branch\"", - "gitops_commit=$(git rev-parse HEAD)", - "git checkout \"$source_branch\" >/dev/null 2>&1 || true", - "CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" node <<'NODE'", - "console.log(JSON.stringify({ ok: true, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));", + " git add source.json \"$artifact_catalog\" \"$gitops_root\"", + " if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi", + " git push -u origin \"$gitops_branch\"", + " gitops_commit=$(git rev-parse HEAD)", + " git checkout \"$source_branch\" >/dev/null 2>&1 || true", + " CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" node <<'NODE' > \"$status_file\"", + "console.log(JSON.stringify({ ok: true, status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));", "NODE", + " trap - EXIT", + ") >\"$stdout_file\" 2>\"$stderr_file\" &", + "pid=$!", + "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +async function waitForYamlLaneGitopsPublish(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise & { ok: boolean; payload: Record }> { + if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "gitops-publish-job-id-missing", valuesPrinted: false } }; + const startedAt = Date.now(); + const timeoutMs = 300_000; + let lastPayload: Record = {}; + let polls = 0; + while (Date.now() - startedAt < timeoutMs) { + polls += 1; + const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishStatusScript(spec, jobId)]); + const payload = captureJsonPayload(probe); + lastPayload = payload; + progressEvent("agentrun.yaml-lane.gitops-publish.progress", { + node: spec.nodeId, + lane: spec.lane, + sourceCommit, + jobId, + polls, + status: stringOrNull(payload.status) ?? "unknown", + gitopsCommit: stringOrNull(payload.gitopsCommit), + elapsedMs: Date.now() - startedAt, + }); + if (payload.ok === true && stringOrNull(payload.gitopsCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; + if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; + await sleep(5_000); + } + return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; +} + +function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string): string { + const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`; + return [ + "set +e", + `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, + "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", ].join("\n"); }