From 1a6b6e3d1f2824d6d944b7d240998c7ab7c9cf54 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 06:15:37 +0000 Subject: [PATCH] fix: poll AgentRun source bootstrap --- config/agentrun.yaml | 6 ++ scripts/src/agentrun-lanes.ts | 9 +++ scripts/src/agentrun.ts | 143 +++++++++++++++++++++++++++------- 3 files changed, 130 insertions(+), 28 deletions(-) diff --git a/config/agentrun.yaml b/config/agentrun.yaml index 769e07c2..7efbe8fd 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -50,6 +50,8 @@ controlPlane: repository: pikasTech/agentrun branch: v0.1 bootstrapFromBranch: v0.1 + bootstrapTimeoutSeconds: 900 + bootstrapPollSeconds: 15 remote: git@github.com:pikasTech/agentrun.git workspace: /root/agentrun-v01 runtime: @@ -115,6 +117,7 @@ controlPlane: - bun.lock - tsconfig.json timeoutSeconds: 1800 + pollSeconds: 15 resources: requests: cpu: 100m @@ -177,6 +180,8 @@ controlPlane: repository: pikasTech/agentrun branch: v0.2 bootstrapFromBranch: v0.1 + bootstrapTimeoutSeconds: 900 + bootstrapPollSeconds: 15 remote: git@github.com:pikasTech/agentrun.git workspace: /home/ubuntu/workspace/agentrun-v02 runtime: @@ -242,6 +247,7 @@ controlPlane: - bun.lock - tsconfig.json timeoutSeconds: 1800 + pollSeconds: 15 resources: requests: cpu: 100m diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index 40812b86..f4014c74 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -33,6 +33,8 @@ export interface AgentRunLaneSpec { readonly repository: string; readonly branch: string; readonly bootstrapFromBranch: string | null; + readonly bootstrapTimeoutSeconds: number; + readonly bootstrapPollSeconds: number; readonly remote: string; readonly workspace: string; }; @@ -143,6 +145,7 @@ export interface AgentRunImageBuildSpec { readonly noProxy: readonly string[]; readonly envIdentityFiles: readonly string[]; readonly timeoutSeconds: number; + readonly pollSeconds: number; } export interface AgentRunLaneTarget { @@ -201,6 +204,8 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record, path: string): AgentRun return item; }), timeoutSeconds: integerField(input, "timeoutSeconds", path), + pollSeconds: integerField(input, "pollSeconds", path), }; } diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 4ba0fba0..27747841 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -2662,6 +2662,8 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt remote: spec.source.remote, branch: spec.source.branch, bootstrapFromBranch: spec.source.bootstrapFromBranch, + bootstrapTimeoutSeconds: spec.source.bootstrapTimeoutSeconds, + bootstrapPollSeconds: spec.source.bootstrapPollSeconds, remoteBranchExists, sourceCommit, }, @@ -2673,6 +2675,7 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt repository: `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}`, containerfile: spec.deployment.manager.imageBuild.containerfile, timeoutSeconds: spec.deployment.manager.imageBuild.timeoutSeconds, + pollSeconds: spec.deployment.manager.imageBuild.pollSeconds, proxyConfigured: spec.deployment.manager.imageBuild.httpProxy !== null || spec.deployment.manager.imageBuild.httpsProxy !== null, }, gitops: { @@ -2711,10 +2714,31 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt } async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise> { - const bootstrap = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapScript(spec)]); - const bootstrapPayload = captureJsonPayload(bootstrap); + progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { + node: spec.nodeId, + lane: spec.lane, + status: "submitting", + }); + const bootstrapSubmit = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapSubmitScript(spec)]); + const bootstrapSubmitPayload = captureJsonPayload(bootstrapSubmit); + if (bootstrapSubmit.exitCode !== 0 || bootstrapSubmitPayload.ok === false) { + return { + ok: false, + command: "agentrun control-plane trigger-current", + mode: waited ? "confirmed-waited" : "confirmed-trigger", + configPath, + target: agentRunLaneSummary(spec), + phase: "source-bootstrap-submit", + degradedReason: "yaml-lane-source-bootstrap-submit-failed", + result: bootstrapSubmitPayload, + capture: compactCapture(bootstrapSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const bootstrap = await waitForYamlLaneSourceBootstrap(config, spec, stringOrNull(bootstrapSubmitPayload.jobId)); + const bootstrapPayload = bootstrap.payload; const sourceCommit = stringOrNull(bootstrapPayload.sourceCommit); - if (bootstrap.exitCode !== 0 || sourceCommit === null || !isGitSha(sourceCommit)) { + if (bootstrap.ok !== true || sourceCommit === null || !isGitSha(sourceCommit)) { return { ok: false, command: "agentrun control-plane trigger-current", @@ -2724,7 +2748,7 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen phase: "source-bootstrap", degradedReason: "yaml-lane-source-bootstrap-failed", result: bootstrapPayload, - capture: compactCapture(bootstrap, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + bootstrapStatus: bootstrap, valuesPrinted: false, }; } @@ -3170,37 +3194,100 @@ function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string { ].join("\n"); } -function yamlLaneSourceBootstrapScript(spec: AgentRunLaneSpec): string { +function yamlLaneSourceBootstrapSubmitScript(spec: AgentRunLaneSpec): string { const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch; + const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; return [ "set -eu", `workspace=${shQuote(spec.source.workspace)}`, `remote=${shQuote(spec.source.remote)}`, `branch=${shQuote(spec.source.branch)}`, `bootstrap_branch=${shQuote(bootstrap)}`, - "mkdir -p \"$(dirname \"$workspace\")\"", - "if [ ! -d \"$workspace/.git\" ]; then", - " git clone --no-checkout \"$remote\" \"$workspace\"", - "fi", - "cd \"$workspace\"", - "git remote set-url origin \"$remote\" || git remote add origin \"$remote\"", - "git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"", - "if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then", - " git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"", - "else", - " git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"", - "fi", - "if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi", - "git add -A deploy/deploy.json 2>/dev/null || true", - "if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then", - " git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'", - "fi", - "git push -u origin \"$branch\"", - "source_commit=$(git rev-parse HEAD)", - "status_short=$(git status --short)", - "SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" node <<'NODE'", - "console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));", + `state_dir=${shQuote(stateDir)}`, + "mkdir -p \"$state_dir\" \"$(dirname \"$workspace\")\"", + "job_id=\"source-bootstrap-$(date +%s)-$$\"", + "status_file=\"$state_dir/$job_id.json\"", + "stdout_file=\"$state_dir/$job_id.stdout.log\"", + "stderr_file=\"$state_dir/$job_id.stderr.log\"", + "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-3000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", + "const code = Number(process.env.CODE || 1);", + "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", "NODE", + " fi; exit \"$code\"; }", + " trap write_failed_status EXIT", + " if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then", + " :", + " else", + " rm -rf \"$workspace\"", + " git clone --no-checkout \"$remote\" \"$workspace\"", + " fi", + " cd \"$workspace\"", + " git remote set-url origin \"$remote\" || git remote add origin \"$remote\"", + " git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"", + " if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then", + " git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"", + " else", + " git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"", + " fi", + " if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi", + " git add -A deploy/deploy.json 2>/dev/null || true", + " if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then", + " git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'", + " fi", + " git push -u origin \"$branch\"", + " source_commit=$(git rev-parse HEAD)", + " status_short=$(git status --short)", + " SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"", + "console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));", + "NODE", + " trap - EXIT", + ") >\"$stdout_file\" 2>\"$stderr_file\" &", + "pid=$!", + "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +async function waitForYamlLaneSourceBootstrap(config: UniDeskConfig, spec: AgentRunLaneSpec, jobId: string | null): Promise & { ok: boolean; payload: Record }> { + if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "source-bootstrap-job-id-missing", valuesPrinted: false } }; + const startedAt = Date.now(); + const timeoutMs = spec.source.bootstrapTimeoutSeconds * 1000; + const pollMs = spec.source.bootstrapPollSeconds * 1000; + let lastPayload: Record = {}; + let polls = 0; + while (Date.now() - startedAt < timeoutMs) { + polls += 1; + const probe = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapStatusScript(spec, jobId)]); + const payload = captureJsonPayload(probe); + lastPayload = payload; + progressEvent("agentrun.yaml-lane.source-bootstrap.progress", { + node: spec.nodeId, + lane: spec.lane, + jobId, + polls, + status: stringOrNull(payload.status) ?? "unknown", + sourceCommit: stringOrNull(payload.sourceCommit), + elapsedMs: Date.now() - startedAt, + }); + if (payload.ok === true && stringOrNull(payload.sourceCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; + if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; + await sleep(pollMs); + } + return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "source-bootstrap-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; +} + +function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: string): string { + const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`; + return [ + "set +e", + `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, + "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", ].join("\n"); } @@ -3295,7 +3382,7 @@ async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLa }); if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; - await sleep(15_000); + await sleep(spec.deployment.manager.imageBuild.pollSeconds * 1000); } return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; }