fix: make AgentRun gitops publish pollable
This commit is contained in:
Codex
+113
-23
@@ -2618,9 +2618,35 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen
|
||||
}
|
||||
const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" });
|
||||
const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image });
|
||||
const gitops = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishScript(spec, renderedFiles)]);
|
||||
const gitopsPayload = captureJsonPayload(gitops);
|
||||
if (gitops.exitCode !== 0 || gitopsPayload.ok === false) {
|
||||
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
sourceCommit,
|
||||
status: "submitting",
|
||||
});
|
||||
const gitopsSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishSubmitScript(spec, renderedFiles)]);
|
||||
const gitopsSubmitPayload = captureJsonPayload(gitopsSubmit);
|
||||
if (gitopsSubmit.exitCode !== 0 || gitopsSubmitPayload.ok === false) {
|
||||
return {
|
||||
ok: false,
|
||||
command: "agentrun control-plane trigger-current",
|
||||
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
||||
configPath,
|
||||
target: agentRunLaneSummary(spec),
|
||||
phase: "gitops-publish-submit",
|
||||
sourceCommit,
|
||||
image,
|
||||
degradedReason: "yaml-lane-gitops-publish-submit-failed",
|
||||
sourceBootstrap: bootstrapPayload,
|
||||
imageBuild: buildPayload,
|
||||
result: gitopsSubmitPayload,
|
||||
capture: compactCapture(gitopsSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
const gitops = await waitForYamlLaneGitopsPublish(config, spec, sourceCommit, stringOrNull(gitopsSubmitPayload.jobId));
|
||||
const gitopsPayload = gitops.payload;
|
||||
if (gitops.ok !== true || gitopsPayload.ok === false) {
|
||||
return {
|
||||
ok: false,
|
||||
command: "agentrun control-plane trigger-current",
|
||||
@@ -2633,8 +2659,9 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen
|
||||
degradedReason: "yaml-lane-gitops-publish-failed",
|
||||
sourceBootstrap: bootstrapPayload,
|
||||
imageBuild: buildPayload,
|
||||
gitopsPublishSubmit: gitopsSubmitPayload,
|
||||
result: gitopsPayload,
|
||||
capture: compactCapture(gitops, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
||||
gitopsPublishStatus: gitops,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
@@ -2671,6 +2698,7 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen
|
||||
sourceBootstrap: bootstrapPayload,
|
||||
imageBuildSubmit: buildSubmitPayload,
|
||||
imageBuild: buildPayload,
|
||||
gitopsPublishSubmit: gitopsSubmitPayload,
|
||||
renderedFiles: {
|
||||
count: renderedFiles.length,
|
||||
digest: renderedFilesDigest(renderedFiles),
|
||||
@@ -3262,13 +3290,15 @@ function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string):
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string {
|
||||
function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string {
|
||||
const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
|
||||
const filesB64 = Buffer.from(JSON.stringify(files.map((file) => ({
|
||||
path: file.path,
|
||||
contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
|
||||
}))), "utf8").toString("base64");
|
||||
return [
|
||||
"set -eu",
|
||||
`state_dir=${shQuote(stateDir)}`,
|
||||
`workspace=${shQuote(spec.source.workspace)}`,
|
||||
`remote=${shQuote(spec.source.remote)}`,
|
||||
`source_branch=${shQuote(spec.source.branch)}`,
|
||||
@@ -3276,17 +3306,34 @@ function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { p
|
||||
`gitops_root=${shQuote(spec.deployment.gitopsRoot)}`,
|
||||
`artifact_catalog=${shQuote(spec.deployment.artifactCatalogPath)}`,
|
||||
`files_b64=${shQuote(filesB64)}`,
|
||||
"cd \"$workspace\"",
|
||||
"git fetch origin \"$gitops_branch\" || true",
|
||||
"if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then",
|
||||
" git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"",
|
||||
"else",
|
||||
" git checkout --orphan \"$gitops_branch\"",
|
||||
" git rm -rf . >/dev/null 2>&1 || true",
|
||||
"fi",
|
||||
"git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true",
|
||||
"rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json",
|
||||
"FILES_B64=\"$files_b64\" node <<'NODE'",
|
||||
"mkdir -p \"$state_dir\"",
|
||||
"job_id=\"gitops-publish-$(date +%s)-$$\"",
|
||||
"status_file=\"$state_dir/$job_id.json\"",
|
||||
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
|
||||
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
|
||||
"cat > \"$status_file\" <<EOF",
|
||||
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"workspace\":\"$workspace\",\"gitopsBranch\":\"$gitops_branch\",\"valuesPrinted\":false}",
|
||||
"EOF",
|
||||
"(",
|
||||
" set -eu",
|
||||
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 80 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" GITOPS_BRANCH=\"$gitops_branch\" node <<'NODE' > \"$status_file\"",
|
||||
"const code = Number(process.env.CODE || 1);",
|
||||
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, gitopsBranch: process.env.GITOPS_BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
" fi; exit \"$code\"; }",
|
||||
" trap write_failed_status EXIT",
|
||||
" cd \"$workspace\"",
|
||||
" git remote set-url origin \"$remote\" || git remote add origin \"$remote\"",
|
||||
" git fetch origin \"$gitops_branch\" || true",
|
||||
" if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then",
|
||||
" git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"",
|
||||
" else",
|
||||
" git checkout --orphan \"$gitops_branch\"",
|
||||
" git rm -rf . >/dev/null 2>&1 || true",
|
||||
" fi",
|
||||
" git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true",
|
||||
" rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json",
|
||||
" FILES_B64=\"$files_b64\" node <<'NODE'",
|
||||
"const fs = require('node:fs');",
|
||||
"const path = require('node:path');",
|
||||
"const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));",
|
||||
@@ -3297,14 +3344,57 @@ function yamlLaneGitopsPublishScript(spec: AgentRunLaneSpec, files: readonly { p
|
||||
" fs.writeFileSync(target, Buffer.from(file.contentBase64, 'base64'));",
|
||||
"}",
|
||||
"NODE",
|
||||
"git add source.json \"$artifact_catalog\" \"$gitops_root\"",
|
||||
"if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi",
|
||||
"git push -u origin \"$gitops_branch\"",
|
||||
"gitops_commit=$(git rev-parse HEAD)",
|
||||
"git checkout \"$source_branch\" >/dev/null 2>&1 || true",
|
||||
"CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" node <<'NODE'",
|
||||
"console.log(JSON.stringify({ ok: true, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));",
|
||||
" git add source.json \"$artifact_catalog\" \"$gitops_root\"",
|
||||
" if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi",
|
||||
" git push -u origin \"$gitops_branch\"",
|
||||
" gitops_commit=$(git rev-parse HEAD)",
|
||||
" git checkout \"$source_branch\" >/dev/null 2>&1 || true",
|
||||
" CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" node <<'NODE' > \"$status_file\"",
|
||||
"console.log(JSON.stringify({ ok: true, status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));",
|
||||
"NODE",
|
||||
" trap - EXIT",
|
||||
") >\"$stdout_file\" 2>\"$stderr_file\" &",
|
||||
"pid=$!",
|
||||
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
|
||||
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function waitForYamlLaneGitopsPublish(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
|
||||
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "gitops-publish-job-id-missing", valuesPrinted: false } };
|
||||
const startedAt = Date.now();
|
||||
const timeoutMs = 300_000;
|
||||
let lastPayload: Record<string, unknown> = {};
|
||||
let polls = 0;
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
polls += 1;
|
||||
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishStatusScript(spec, jobId)]);
|
||||
const payload = captureJsonPayload(probe);
|
||||
lastPayload = payload;
|
||||
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
sourceCommit,
|
||||
jobId,
|
||||
polls,
|
||||
status: stringOrNull(payload.status) ?? "unknown",
|
||||
gitopsCommit: stringOrNull(payload.gitopsCommit),
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
});
|
||||
if (payload.ok === true && stringOrNull(payload.gitopsCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
|
||||
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
|
||||
await sleep(5_000);
|
||||
}
|
||||
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
|
||||
}
|
||||
|
||||
function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
|
||||
const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
|
||||
return [
|
||||
"set +e",
|
||||
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
|
||||
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user