fix: poll AgentRun source bootstrap
This commit is contained in:
Codex
@@ -50,6 +50,8 @@ controlPlane:
|
||||
repository: pikasTech/agentrun
|
||||
branch: v0.1
|
||||
bootstrapFromBranch: v0.1
|
||||
bootstrapTimeoutSeconds: 900
|
||||
bootstrapPollSeconds: 15
|
||||
remote: git@github.com:pikasTech/agentrun.git
|
||||
workspace: /root/agentrun-v01
|
||||
runtime:
|
||||
@@ -115,6 +117,7 @@ controlPlane:
|
||||
- bun.lock
|
||||
- tsconfig.json
|
||||
timeoutSeconds: 1800
|
||||
pollSeconds: 15
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
@@ -177,6 +180,8 @@ controlPlane:
|
||||
repository: pikasTech/agentrun
|
||||
branch: v0.2
|
||||
bootstrapFromBranch: v0.1
|
||||
bootstrapTimeoutSeconds: 900
|
||||
bootstrapPollSeconds: 15
|
||||
remote: git@github.com:pikasTech/agentrun.git
|
||||
workspace: /home/ubuntu/workspace/agentrun-v02
|
||||
runtime:
|
||||
@@ -242,6 +247,7 @@ controlPlane:
|
||||
- bun.lock
|
||||
- tsconfig.json
|
||||
timeoutSeconds: 1800
|
||||
pollSeconds: 15
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
|
||||
@@ -33,6 +33,8 @@ export interface AgentRunLaneSpec {
|
||||
readonly repository: string;
|
||||
readonly branch: string;
|
||||
readonly bootstrapFromBranch: string | null;
|
||||
readonly bootstrapTimeoutSeconds: number;
|
||||
readonly bootstrapPollSeconds: number;
|
||||
readonly remote: string;
|
||||
readonly workspace: string;
|
||||
};
|
||||
@@ -143,6 +145,7 @@ export interface AgentRunImageBuildSpec {
|
||||
readonly noProxy: readonly string[];
|
||||
readonly envIdentityFiles: readonly string[];
|
||||
readonly timeoutSeconds: number;
|
||||
readonly pollSeconds: number;
|
||||
}
|
||||
|
||||
export interface AgentRunLaneTarget {
|
||||
@@ -201,6 +204,8 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record<string, unkn
|
||||
repository: spec.source.repository,
|
||||
branch: spec.source.branch,
|
||||
bootstrapFromBranch: spec.source.bootstrapFromBranch,
|
||||
bootstrapTimeoutSeconds: spec.source.bootstrapTimeoutSeconds,
|
||||
bootstrapPollSeconds: spec.source.bootstrapPollSeconds,
|
||||
workspace: spec.source.workspace,
|
||||
},
|
||||
runtime: {
|
||||
@@ -246,6 +251,7 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record<string, unkn
|
||||
noProxyCount: spec.deployment.manager.imageBuild.noProxy.length,
|
||||
envIdentityFileCount: spec.deployment.manager.imageBuild.envIdentityFiles.length,
|
||||
timeoutSeconds: spec.deployment.manager.imageBuild.timeoutSeconds,
|
||||
pollSeconds: spec.deployment.manager.imageBuild.pollSeconds,
|
||||
},
|
||||
resources: spec.deployment.manager.resources,
|
||||
},
|
||||
@@ -367,6 +373,8 @@ function parseLane(lane: string, node: AgentRunNodeSpec, input: Record<string, u
|
||||
repository: stringField(source, "repository", `${path}.source`),
|
||||
branch: stringField(source, "branch", `${path}.source`),
|
||||
bootstrapFromBranch: optionalStringField(source, "bootstrapFromBranch", `${path}.source`) ?? null,
|
||||
bootstrapTimeoutSeconds: integerField(source, "bootstrapTimeoutSeconds", `${path}.source`),
|
||||
bootstrapPollSeconds: integerField(source, "bootstrapPollSeconds", `${path}.source`),
|
||||
remote: stringField(source, "remote", `${path}.source`),
|
||||
workspace: absolutePathField(source, "workspace", `${path}.source`),
|
||||
},
|
||||
@@ -495,6 +503,7 @@ function parseImageBuild(input: Record<string, unknown>, path: string): AgentRun
|
||||
return item;
|
||||
}),
|
||||
timeoutSeconds: integerField(input, "timeoutSeconds", path),
|
||||
pollSeconds: integerField(input, "pollSeconds", path),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+115
-28
@@ -2662,6 +2662,8 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt
|
||||
remote: spec.source.remote,
|
||||
branch: spec.source.branch,
|
||||
bootstrapFromBranch: spec.source.bootstrapFromBranch,
|
||||
bootstrapTimeoutSeconds: spec.source.bootstrapTimeoutSeconds,
|
||||
bootstrapPollSeconds: spec.source.bootstrapPollSeconds,
|
||||
remoteBranchExists,
|
||||
sourceCommit,
|
||||
},
|
||||
@@ -2673,6 +2675,7 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt
|
||||
repository: `${spec.ci.registryPrefix}/${spec.deployment.manager.imageBuild.repository}`,
|
||||
containerfile: spec.deployment.manager.imageBuild.containerfile,
|
||||
timeoutSeconds: spec.deployment.manager.imageBuild.timeoutSeconds,
|
||||
pollSeconds: spec.deployment.manager.imageBuild.pollSeconds,
|
||||
proxyConfigured: spec.deployment.manager.imageBuild.httpProxy !== null || spec.deployment.manager.imageBuild.httpsProxy !== null,
|
||||
},
|
||||
gitops: {
|
||||
@@ -2711,10 +2714,31 @@ async function triggerCurrentYamlLane(config: UniDeskConfig, options: TriggerOpt
|
||||
}
|
||||
|
||||
async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: AgentRunLaneSpec, configPath: string, waited: boolean): Promise<Record<string, unknown>> {
|
||||
const bootstrap = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapScript(spec)]);
|
||||
const bootstrapPayload = captureJsonPayload(bootstrap);
|
||||
progressEvent("agentrun.yaml-lane.source-bootstrap.progress", {
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
status: "submitting",
|
||||
});
|
||||
const bootstrapSubmit = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapSubmitScript(spec)]);
|
||||
const bootstrapSubmitPayload = captureJsonPayload(bootstrapSubmit);
|
||||
if (bootstrapSubmit.exitCode !== 0 || bootstrapSubmitPayload.ok === false) {
|
||||
return {
|
||||
ok: false,
|
||||
command: "agentrun control-plane trigger-current",
|
||||
mode: waited ? "confirmed-waited" : "confirmed-trigger",
|
||||
configPath,
|
||||
target: agentRunLaneSummary(spec),
|
||||
phase: "source-bootstrap-submit",
|
||||
degradedReason: "yaml-lane-source-bootstrap-submit-failed",
|
||||
result: bootstrapSubmitPayload,
|
||||
capture: compactCapture(bootstrapSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
const bootstrap = await waitForYamlLaneSourceBootstrap(config, spec, stringOrNull(bootstrapSubmitPayload.jobId));
|
||||
const bootstrapPayload = bootstrap.payload;
|
||||
const sourceCommit = stringOrNull(bootstrapPayload.sourceCommit);
|
||||
if (bootstrap.exitCode !== 0 || sourceCommit === null || !isGitSha(sourceCommit)) {
|
||||
if (bootstrap.ok !== true || sourceCommit === null || !isGitSha(sourceCommit)) {
|
||||
return {
|
||||
ok: false,
|
||||
command: "agentrun control-plane trigger-current",
|
||||
@@ -2724,7 +2748,7 @@ async function triggerCurrentYamlLaneConfirmed(config: UniDeskConfig, spec: Agen
|
||||
phase: "source-bootstrap",
|
||||
degradedReason: "yaml-lane-source-bootstrap-failed",
|
||||
result: bootstrapPayload,
|
||||
capture: compactCapture(bootstrap, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
|
||||
bootstrapStatus: bootstrap,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
@@ -3170,37 +3194,100 @@ function yamlLaneSourceBootstrapProbeScript(spec: AgentRunLaneSpec): string {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function yamlLaneSourceBootstrapScript(spec: AgentRunLaneSpec): string {
|
||||
function yamlLaneSourceBootstrapSubmitScript(spec: AgentRunLaneSpec): string {
|
||||
const bootstrap = spec.source.bootstrapFromBranch ?? spec.source.branch;
|
||||
const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`;
|
||||
return [
|
||||
"set -eu",
|
||||
`workspace=${shQuote(spec.source.workspace)}`,
|
||||
`remote=${shQuote(spec.source.remote)}`,
|
||||
`branch=${shQuote(spec.source.branch)}`,
|
||||
`bootstrap_branch=${shQuote(bootstrap)}`,
|
||||
"mkdir -p \"$(dirname \"$workspace\")\"",
|
||||
"if [ ! -d \"$workspace/.git\" ]; then",
|
||||
" git clone --no-checkout \"$remote\" \"$workspace\"",
|
||||
"fi",
|
||||
"cd \"$workspace\"",
|
||||
"git remote set-url origin \"$remote\" || git remote add origin \"$remote\"",
|
||||
"git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"",
|
||||
"if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then",
|
||||
" git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"",
|
||||
"else",
|
||||
" git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"",
|
||||
"fi",
|
||||
"if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi",
|
||||
"git add -A deploy/deploy.json 2>/dev/null || true",
|
||||
"if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then",
|
||||
" git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'",
|
||||
"fi",
|
||||
"git push -u origin \"$branch\"",
|
||||
"source_commit=$(git rev-parse HEAD)",
|
||||
"status_short=$(git status --short)",
|
||||
"SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" node <<'NODE'",
|
||||
"console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));",
|
||||
`state_dir=${shQuote(stateDir)}`,
|
||||
"mkdir -p \"$state_dir\" \"$(dirname \"$workspace\")\"",
|
||||
"job_id=\"source-bootstrap-$(date +%s)-$$\"",
|
||||
"status_file=\"$state_dir/$job_id.json\"",
|
||||
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
|
||||
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
|
||||
"cat > \"$status_file\" <<EOF",
|
||||
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"workspace\":\"$workspace\",\"branch\":\"$branch\",\"valuesPrinted\":false}",
|
||||
"EOF",
|
||||
"(",
|
||||
" set -eu",
|
||||
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 60 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-3000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"",
|
||||
"const code = Number(process.env.CODE || 1);",
|
||||
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
" fi; exit \"$code\"; }",
|
||||
" trap write_failed_status EXIT",
|
||||
" if [ -d \"$workspace/.git\" ] && git -C \"$workspace\" rev-parse --git-dir >/dev/null 2>&1; then",
|
||||
" :",
|
||||
" else",
|
||||
" rm -rf \"$workspace\"",
|
||||
" git clone --no-checkout \"$remote\" \"$workspace\"",
|
||||
" fi",
|
||||
" cd \"$workspace\"",
|
||||
" git remote set-url origin \"$remote\" || git remote add origin \"$remote\"",
|
||||
" git fetch origin \"$bootstrap_branch\" \"$branch\" || git fetch origin \"$bootstrap_branch\"",
|
||||
" if git rev-parse --verify \"refs/remotes/origin/$branch^{commit}\" >/dev/null 2>&1; then",
|
||||
" git checkout -B \"$branch\" \"refs/remotes/origin/$branch\"",
|
||||
" else",
|
||||
" git checkout -B \"$branch\" \"refs/remotes/origin/$bootstrap_branch\"",
|
||||
" fi",
|
||||
" if [ -f deploy/deploy.json ]; then rm deploy/deploy.json; fi",
|
||||
" git add -A deploy/deploy.json 2>/dev/null || true",
|
||||
" if ! git diff --quiet --cached -- deploy/deploy.json 2>/dev/null; then",
|
||||
" git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m 'chore: remove service deploy json truth'",
|
||||
" fi",
|
||||
" git push -u origin \"$branch\"",
|
||||
" source_commit=$(git rev-parse HEAD)",
|
||||
" status_short=$(git status --short)",
|
||||
" SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" BRANCH=\"$branch\" node <<'NODE' > \"$status_file\"",
|
||||
"console.log(JSON.stringify({ ok: process.env.STATUS_SHORT === '', status: 'succeeded', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, branch: process.env.BRANCH, sourceCommit: process.env.SOURCE_COMMIT, workspaceClean: process.env.STATUS_SHORT === '', statusShort: process.env.STATUS_SHORT || null, removedServiceDeployJson: true, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
" trap - EXIT",
|
||||
") >\"$stdout_file\" 2>\"$stderr_file\" &",
|
||||
"pid=$!",
|
||||
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
|
||||
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function waitForYamlLaneSourceBootstrap(config: UniDeskConfig, spec: AgentRunLaneSpec, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
|
||||
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "source-bootstrap-job-id-missing", valuesPrinted: false } };
|
||||
const startedAt = Date.now();
|
||||
const timeoutMs = spec.source.bootstrapTimeoutSeconds * 1000;
|
||||
const pollMs = spec.source.bootstrapPollSeconds * 1000;
|
||||
let lastPayload: Record<string, unknown> = {};
|
||||
let polls = 0;
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
polls += 1;
|
||||
const probe = await capture(config, spec.nodeRoute, ["script", "--", yamlLaneSourceBootstrapStatusScript(spec, jobId)]);
|
||||
const payload = captureJsonPayload(probe);
|
||||
lastPayload = payload;
|
||||
progressEvent("agentrun.yaml-lane.source-bootstrap.progress", {
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
jobId,
|
||||
polls,
|
||||
status: stringOrNull(payload.status) ?? "unknown",
|
||||
sourceCommit: stringOrNull(payload.sourceCommit),
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
});
|
||||
if (payload.ok === true && stringOrNull(payload.sourceCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
|
||||
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
|
||||
await sleep(pollMs);
|
||||
}
|
||||
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "source-bootstrap-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
|
||||
}
|
||||
|
||||
function yamlLaneSourceBootstrapStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
|
||||
const stateDir = `/tmp/unidesk-agentrun-source-${spec.nodeId}-${spec.lane}`;
|
||||
return [
|
||||
"set +e",
|
||||
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
|
||||
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -3295,7 +3382,7 @@ async function waitForYamlLaneBuildImage(config: UniDeskConfig, spec: AgentRunLa
|
||||
});
|
||||
if (payload.ok === true && stringOrNull(payload.digest) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
|
||||
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
|
||||
await sleep(15_000);
|
||||
await sleep(spec.deployment.manager.imageBuild.pollSeconds * 1000);
|
||||
}
|
||||
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "image-build-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user