mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复 AgentRun unreachable 泛化与 YAML-lane GitOps 发布分叉 (#624)

Browse Source 
* fix: expose web-probe diagnostic previews

* fix: classify agentrun unreachable root causes

---------

Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-22 03:26:28 +08:00
committed by GitHub
parent b1ae0b6408
commit d44e195ae8
2 changed files with 289 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/agentrun.ts
+237 -126
View File
@@ -2998,33 +2998,31 @@ async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec:
}
const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" });
const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image });
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
node: spec.nodeId,
lane: spec.lane,
sourceCommit,
status: "submitting",
});
const gitopsSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishSubmitScript(spec, renderedFiles)]);
const gitopsSubmitPayload = captureJsonPayload(gitopsSubmit);
if (gitopsSubmit.exitCode !== 0 || gitopsSubmitPayload.ok === false) {
const mirrorPrePublish = await runYamlLaneGitMirrorSyncJob(config, spec);
if (mirrorPrePublish.ok !== true) {
return {
ok: false,
command: "agentrun control-plane trigger-current",
mode: waited ? "confirmed-waited" : "confirmed-trigger",
configPath,
target: agentRunLaneSummary(spec),
phase: "gitops-publish-submit",
phase: "git-mirror-prepublish-sync",
sourceCommit,
image,
degradedReason: "yaml-lane-gitops-publish-submit-failed",
degradedReason: "yaml-lane-git-mirror-prepublish-sync-failed",
sourceBootstrap: bootstrapPayload,
imageBuild: buildPayload,
result: gitopsSubmitPayload,
capture: compactCapture(gitopsSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
result: mirrorPrePublish,
valuesPrinted: false,
};
}
const gitops = await waitForYamlLaneGitopsPublish(config, spec, sourceCommit, stringOrNull(gitopsSubmitPayload.jobId));
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
node: spec.nodeId,
lane: spec.lane,
sourceCommit,
status: "submitting",
});
const gitops = await runYamlLaneGitopsPublishJob(config, spec, sourceCommit, renderedFiles);
const gitopsPayload = gitops.payload;
if (gitops.ok !== true || gitopsPayload.ok === false) {
return {
@@ -3039,26 +3037,27 @@ async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec:
degradedReason: "yaml-lane-gitops-publish-failed",
sourceBootstrap: bootstrapPayload,
imageBuild: buildPayload,
gitopsPublishSubmit: gitopsSubmitPayload,
gitMirrorPrePublish: mirrorPrePublish,
result: gitopsPayload,
gitopsPublishStatus: gitops,
valuesPrinted: false,
};
}
const mirror = await runYamlLaneGitMirrorSyncJob(config, spec);
if (mirror.ok !== true) {
const mirrorFlush = await runYamlLaneGitMirrorFlushJob(config, spec);
if (mirrorFlush.ok !== true) {
return {
ok: false,
command: "agentrun control-plane trigger-current",
mode: waited ? "confirmed-waited" : "confirmed-trigger",
configPath,
target: agentRunLaneSummary(spec),
phase: "git-mirror-sync",
phase: "git-mirror-flush",
sourceCommit,
image,
gitops: gitopsPayload,
degradedReason: "yaml-lane-git-mirror-sync-failed",
result: mirror,
gitMirrorPrePublish: mirrorPrePublish,
degradedReason: "yaml-lane-git-mirror-flush-failed",
result: mirrorFlush,
valuesPrinted: false,
};
}
@@ -3078,13 +3077,13 @@ async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec:
sourceBootstrap: bootstrapPayload,
imageBuildSubmit: buildSubmitPayload,
imageBuild: buildPayload,
gitopsPublishSubmit: gitopsSubmitPayload,
renderedFiles: {
count: renderedFiles.length,
digest: renderedFilesDigest(renderedFiles),
},
gitops: gitopsPayload,
gitMirror: mirror,
gitMirrorPrePublish: mirrorPrePublish,
gitMirror: mirrorFlush,
created: createPayload,
capture: compactCapture(created, { full: created.exitCode !== 0, stdoutTailChars: 4000, stderrTailChars: 4000 }),
next: {
@@ -3853,52 +3852,175 @@ function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string):
].join("\n");
}
function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string {
const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
async function runYamlLaneGitopsPublishJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, files: readonly { path: string; content: string }[]): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
const jobName = `gitops-publish-${spec.nodeId.toLowerCase()}-${spec.lane}-${Date.now().toString(36)}`.slice(0, 63);
const manifest = yamlLaneGitopsPublishJobManifest(spec, files, jobName);
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
if (created.exitCode !== 0) {
return {
ok: false,
payload: { ok: false, status: "create-failed", jobName, degradedReason: "gitops-publish-job-create-failed", valuesPrinted: false },
jobName,
sourceCommit,
phase: "create-job",
capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
const startedAt = Date.now();
const timeoutMs = 300_000;
let polls = 0;
let lastPayload: Record<string, unknown> = {};
let lastProbe: SshCaptureResult | null = null;
while (Date.now() - startedAt < timeoutMs) {
polls += 1;
lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]);
const probePayload = captureJsonPayload(lastProbe);
const publishPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload);
if (Object.keys(publishPayload).length > 0) lastPayload = publishPayload;
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
node: spec.nodeId,
lane: spec.lane,
sourceCommit,
jobName,
polls,
status: stringOrNull(publishPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"),
gitopsCommit: stringOrNull(publishPayload.gitopsCommit),
elapsedMs: Date.now() - startedAt,
});
if (probePayload.succeeded === true) {
if (publishPayload.ok === true && stringOrNull(publishPayload.gitopsCommit) !== null) {
return { ok: true, payload: publishPayload, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false };
}
return {
ok: false,
payload: { ...publishPayload, ok: false, status: stringOrNull(publishPayload.status) ?? "succeeded-without-result", degradedReason: "gitops-publish-result-missing", jobName, valuesPrinted: false },
jobName,
polls,
elapsedMs: Date.now() - startedAt,
probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
if (probePayload.failed === true) {
const payload = Object.keys(publishPayload).length > 0 ? publishPayload : { ok: false, status: "failed", degradedReason: "gitops-publish-job-failed", jobName, valuesPrinted: false };
return {
ok: false,
payload,
jobName,
polls,
elapsedMs: Date.now() - startedAt,
probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
await sleep(5_000);
}
return {
ok: false,
payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-job-timeout", jobName, valuesPrinted: false },
jobName,
polls,
elapsedMs: Date.now() - startedAt,
probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
valuesPrinted: false,
};
}
function yamlLaneGitopsPublishPayloadFromProbe(probePayload: Record<string, unknown>): Record<string, unknown> {
const logsTail = stringOrNull(probePayload.logsTail);
if (logsTail === null) return {};
const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
for (let index = lines.length - 1; index >= 0; index -= 1) {
const line = lines[index];
if (!line.startsWith("{") || !line.endsWith("}")) continue;
try {
const parsed = JSON.parse(line) as unknown;
const payload = record(parsed);
if (payload.ok === true || payload.ok === false) return payload;
} catch {
continue;
}
}
return {};
}
function yamlLaneGitopsPublishJobManifest(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[], name: string): Record<string, unknown> {
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: {
name,
namespace: spec.gitMirror.namespace,
labels: {
"app.kubernetes.io/name": "gitops-publish",
"app.kubernetes.io/part-of": "agentrun",
"agentrun.pikastech.local/lane": spec.version,
"agentrun.pikastech.local/node": spec.nodeId,
"agentrun.pikastech.local/component": "gitops-publish",
},
},
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 600,
ttlSecondsAfterFinished: 3600,
template: {
metadata: {
labels: {
"app.kubernetes.io/name": "gitops-publish",
"app.kubernetes.io/part-of": "agentrun",
"agentrun.pikastech.local/lane": spec.version,
"agentrun.pikastech.local/node": spec.nodeId,
"agentrun.pikastech.local/component": "gitops-publish",
},
},
spec: {
restartPolicy: "Never",
volumes: [yamlLaneGitMirrorCacheVolume(spec)],
containers: [{
name: "publish",
image: spec.gitMirror.toolsImage,
imagePullPolicy: "IfNotPresent",
command: ["/bin/sh", "-ec", yamlLaneGitopsPublishShell(spec, files, name)],
volumeMounts: [{ name: "cache", mountPath: "/cache" }],
}],
},
},
},
};
}
function yamlLaneGitopsPublishShell(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[], jobName: string): string {
const filesB64 = Buffer.from(JSON.stringify(files.map((file) => ({
path: file.path,
contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
}))), "utf8").toString("base64");
return [
"set -eu",
`state_dir=${shQuote(stateDir)}`,
`remote=${shQuote(spec.source.remote)}`,
`repository=${shQuote(spec.source.repository)}`,
`gitops_branch=${shQuote(spec.gitops.branch)}`,
`gitops_root=${shQuote(spec.deployment.gitopsRoot)}`,
`artifact_catalog=${shQuote(spec.deployment.artifactCatalogPath)}`,
`files_b64=${shQuote(filesB64)}`,
"mkdir -p \"$state_dir\"",
"job_id=\"gitops-publish-$(date +%s)-$$\"",
"status_file=\"$state_dir/$job_id.json\"",
"stdout_file=\"$state_dir/$job_id.stdout.log\"",
"stderr_file=\"$state_dir/$job_id.stderr.log\"",
"publish_root=\"$state_dir/worktrees\"",
"publish_worktree=\"$publish_root/$job_id\"",
"cat > \"$status_file\" <<EOF",
"{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"publishWorkspace\":\"$publish_worktree\",\"gitopsBranch\":\"$gitops_branch\",\"valuesPrinted\":false}",
"EOF",
"(",
" set -eu",
" write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 80 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" PUBLISH_WORKSPACE=\"$publish_worktree\" GITOPS_BRANCH=\"$gitops_branch\" node <<'NODE' > \"$status_file\"",
"const code = Number(process.env.CODE || 1);",
"console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, publishWorkspace: process.env.PUBLISH_WORKSPACE, gitopsBranch: process.env.GITOPS_BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
"NODE",
" fi; exit \"$code\"; }",
" trap write_failed_status EXIT",
" mkdir -p \"$publish_root\"",
" rm -rf \"$publish_worktree\"",
" git clone --no-checkout \"$remote\" \"$publish_worktree\"",
" cd \"$publish_worktree\"",
" git fetch origin \"$gitops_branch\" || true",
" if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then",
" git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"",
" else",
" git checkout --orphan \"$gitops_branch\"",
" git rm -rf . >/dev/null 2>&1 || true",
" fi",
" git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true",
" rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json",
" FILES_B64=\"$files_b64\" node <<'NODE'",
`job_name=${shQuote(jobName)}`,
"remote=\"/cache/${repository}.git\"",
"remote_mode=git-mirror-cache-volume",
"publish_worktree=\"/tmp/$job_name\"",
"write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then printf '%s\\n' \"{\\\"ok\\\":false,\\\"status\\\":\\\"failed\\\",\\\"exitCode\\\":$code,\\\"jobName\\\":\\\"$job_name\\\",\\\"publishWorkspace\\\":\\\"$publish_worktree\\\",\\\"gitopsBranch\\\":\\\"$gitops_branch\\\",\\\"publishRemoteMode\\\":\\\"$remote_mode\\\",\\\"valuesPrinted\\\":false}\"; fi; exit \"$code\"; }",
"trap write_failed_status EXIT",
"rm -rf \"$publish_worktree\"",
"git clone --no-checkout \"$remote\" \"$publish_worktree\"",
"cd \"$publish_worktree\"",
"git fetch origin \"$gitops_branch\" || true",
"if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then",
" git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"",
"else",
" git checkout --orphan \"$gitops_branch\"",
" git rm -rf . >/dev/null 2>&1 || true",
"fi",
"git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true",
"rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json",
"FILES_B64=\"$files_b64\" node <<'NODE'",
"const fs = require('node:fs');",
"const path = require('node:path');",
"const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));",
@@ -3909,57 +4031,14 @@ function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readon
" fs.writeFileSync(target, Buffer.from(file.contentBase64, 'base64'));",
"}",
"NODE",
" git add source.json \"$artifact_catalog\" \"$gitops_root\"",
" if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi",
" git push -u origin \"$gitops_branch\"",
" gitops_commit=$(git rev-parse HEAD)",
" rm -rf \"$publish_worktree\"",
" CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_ID=\"$job_id\" node <<'NODE' > \"$status_file\"",
"console.log(JSON.stringify({ ok: true, status: 'succeeded', jobId: process.env.JOB_ID, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));",
"git add source.json \"$artifact_catalog\" \"$gitops_root\"",
"if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi",
"git push -u origin \"$gitops_branch\"",
"gitops_commit=$(git rev-parse HEAD)",
"CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_NAME=\"$job_name\" PUBLISH_REMOTE_MODE=\"$remote_mode\" node <<'NODE'",
"console.log(JSON.stringify({ ok: true, status: 'succeeded', jobName: process.env.JOB_NAME, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, publishRemoteMode: process.env.PUBLISH_REMOTE_MODE || null, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));",
"NODE",
" trap - EXIT",
") >\"$stdout_file\" 2>\"$stderr_file\" &",
"pid=$!",
"JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
"console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
"NODE",
].join("\n");
}
async function waitForYamlLaneGitopsPublish(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "gitops-publish-job-id-missing", valuesPrinted: false } };
const startedAt = Date.now();
const timeoutMs = 300_000;
let lastPayload: Record<string, unknown> = {};
let polls = 0;
while (Date.now() - startedAt < timeoutMs) {
polls += 1;
const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishStatusScript(spec, jobId)]);
const payload = captureJsonPayload(probe);
lastPayload = payload;
progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
node: spec.nodeId,
lane: spec.lane,
sourceCommit,
jobId,
polls,
status: stringOrNull(payload.status) ?? "unknown",
gitopsCommit: stringOrNull(payload.gitopsCommit),
elapsedMs: Date.now() - startedAt,
});
if (payload.ok === true && stringOrNull(payload.gitopsCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
await sleep(5_000);
}
return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
}
function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
return [
"set +e",
`status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
"if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
"trap - EXIT",
].join("\n");
}
@@ -4028,14 +4107,23 @@ function agentRunGitMirrorRetrySummary(attempts: Record<string, unknown>[]): Rec
}
async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise<Record<string, unknown>> {
return await runYamlLaneGitMirrorJob(config, spec, "sync");
}
async function runYamlLaneGitMirrorFlushJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise<Record<string, unknown>> {
return await runYamlLaneGitMirrorJob(config, spec, "flush");
}
async function runYamlLaneGitMirrorJob(config: UniDeskConfig, spec: AgentRunLaneSpec, action: "sync" | "flush"): Promise<Record<string, unknown>> {
const attempts: Record<string, unknown>[] = [];
for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) {
const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63);
const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName);
const jobPrefix = action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix;
const jobName = `${jobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63);
const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName);
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
if (created.exitCode !== 0) {
const result = { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false };
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result);
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
@@ -4054,6 +4142,7 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun
progressEvent("agentrun.yaml-lane.git-mirror.progress", {
node: spec.nodeId,
lane: spec.lane,
action,
jobName,
polls,
retryAttempt: attempt,
@@ -4069,7 +4158,7 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun
}
if (payload.failed === true) {
const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result);
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
@@ -4081,9 +4170,10 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun
await sleep(5_000);
}
if (attempts.length > 0 && record(attempts[attempts.length - 1]).jobName === jobName) continue;
const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
const degradedReason = `git-mirror-${action}-job-timeout`;
const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason, capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result);
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
await sleep(agentRunGitMirrorRetryDelayMs(attempt));
@@ -4091,7 +4181,7 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun
}
return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) };
}
return { ok: false, degradedReason: "git-mirror-sync-retry-loop-exhausted", retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false };
return { ok: false, degradedReason: `git-mirror-${action}-retry-loop-exhausted`, retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false };
}
function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCommit: string, pipelineRun: string): string {
@@ -6000,7 +6090,8 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
}
response = await fetch(new URL(pathValue, clientConfig.manager.baseUrl), init);
} catch (error) {
throw new AgentRunRestError("agentrun-unreachable", `AgentRun server is unreachable for ${method} ${pathValue}: ${error instanceof Error ? error.message : String(error)}`, { bridge: { ...bridgeBase, elapsedMs: Date.now() - startedAt } });
const timedOut = isAbortLikeError(error);
throw new AgentRunRestError(timedOut ? "agentrun-timeout" : "agentrun-unreachable", timedOut ? `AgentRun server request timed out for ${method} ${pathValue} after ${clientConfig.manager.timeoutMs}ms` : `AgentRun server is unreachable for ${method} ${pathValue}: ${error instanceof Error ? error.message : String(error)}`, { bridge: { ...bridgeBase, elapsedMs: Date.now() - startedAt } });
} finally {
clearTimeout(timeout);
}
@@ -6032,14 +6123,21 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
};
}
function isAbortLikeError(error: unknown): boolean {
if (!(error instanceof Error)) return false;
return error.name === "AbortError" || /abort|timed out|timeout/iu.test(error.message);
}
async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body: unknown, clientConfig: AgentRunClientConfig, target: AgentRunRestTarget): Promise<Record<string, unknown>> {
const bridgeBase = agentRunLaneRestBridgeMetadata(clientConfig, target, method, pathValue);
const startedAt = Date.now();
const proxyBaseUrl = `http://127.0.0.1:${target.spec.runtime.managerPort}`;
const request = {
method,
path: pathValue,
body: body ?? null,
baseUrl: target.spec.runtime.internalBaseUrl,
baseUrl: proxyBaseUrl,
serviceBaseUrl: target.spec.runtime.internalBaseUrl,
timeoutMs: clientConfig.manager.timeoutMs,
authEnv: clientConfig.auth.env,
header: clientConfig.auth.header,
@@ -6054,7 +6152,9 @@ async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMeth
}
const proxy = captureJsonPayload(captureResult);
if (proxy.ok !== true) {
throw new AgentRunRestError("agentrun-unreachable", stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "valuesPrinted"]) });
const proxyFailureKind = stringOrNull(proxy.failureKind);
const failureKind: AgentRunFailureKind = proxyFailureKind === "manager-pod-fetch-timeout" ? "agentrun-timeout" : "agentrun-unreachable";
throw new AgentRunRestError(failureKind, stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "path", "baseUrl", "valuesPrinted"]) });
}
const httpStatus = nonNegativeIntegerOrNull(proxy.httpStatus) ?? 0;
const text = stringOrNull(proxy.text) ?? "";
@@ -6089,8 +6189,17 @@ function agentRunLaneRestProxyScript(spec: AgentRunLaneSpec, request: Record<str
const requestB64 = Buffer.from(JSON.stringify(request), "utf8").toString("base64");
const evalScript = String.raw`
const startedAt = Date.now();
let input = {};
let method = "GET";
let path = "/";
let baseUrl = "http://127.0.0.1:8080";
let timeoutMs = 15000;
try {
const input = JSON.parse(Buffer.from(process.env.AGENTRUN_LANE_REST_REQUEST_B64 || "", "base64").toString("utf8"));
input = JSON.parse(Buffer.from(process.env.AGENTRUN_LANE_REST_REQUEST_B64 || "", "base64").toString("utf8"));
method = String(input.method || "GET");
path = String(input.path || "/");
baseUrl = String(input.baseUrl || "http://127.0.0.1:8080");
timeoutMs = Number(input.timeoutMs || 15000);
const headers = {};
const authEnv = String(input.authEnv || "HWLAB_API_KEY");
const apiKey = process.env[authEnv] || process.env.AGENTRUN_API_KEY || process.env.HWLAB_API_KEY || "";
@@ -6101,12 +6210,11 @@ try {
body = JSON.stringify(input.body);
}
const controller = new AbortController();
const timeoutMs = Number(input.timeoutMs || 15000);
const timeout = setTimeout(() => controller.abort(), timeoutMs);
let response;
try {
response = await fetch(new URL(String(input.path || "/"), String(input.baseUrl || "http://127.0.0.1:8080")).toString(), {
method: String(input.method || "GET"),
response = await fetch(new URL(path, baseUrl).toString(), {
method,
headers,
body,
signal: controller.signal,
@@ -6117,7 +6225,10 @@ try {
const text = await response.text();
console.log(JSON.stringify({ ok: true, httpStatus: response.status, statusText: response.statusText, text, elapsedMs: Date.now() - startedAt, valuesPrinted: false }));
} catch (error) {
console.log(JSON.stringify({ ok: false, failureKind: "manager-pod-fetch-failed", message: error instanceof Error ? error.message : String(error), elapsedMs: Date.now() - startedAt, valuesPrinted: false }));
const name = error instanceof Error ? error.name : "";
const message = error instanceof Error ? error.message : String(error);
const timedOut = name === "AbortError" || /abort|timed out|timeout/iu.test(message);
console.log(JSON.stringify({ ok: false, failureKind: timedOut ? "manager-pod-fetch-timeout" : "manager-pod-fetch-failed", message: timedOut ? "AgentRun manager request timed out for " + method + " " + path + " after " + timeoutMs + "ms" : message, elapsedMs: Date.now() - startedAt, path, baseUrl, valuesPrinted: false }));
}
`;
return [
@@ -6733,7 +6844,7 @@ function safeAgentRunEnvelope(envelope: Record<string, unknown>): Record<string,
}
function normalizeAgentRunFailureKind(raw: string | null, httpStatus: number): AgentRunFailureKind {
if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed" || raw === "not-found") return raw;
if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "agentrun-timeout" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed" || raw === "not-found") return raw;
if (httpStatus === 401 || httpStatus === 403) return "auth-failed";
if (httpStatus === 404) return "not-found";
return raw === "schema-invalid" ? "validation-failed" : "validation-failed";
@@ -6823,7 +6934,7 @@ interface AgentRunResolvedAuth {
}
type AgentRunHttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "schema-mismatch" | "unsupported-version" | "validation-failed" | "not-found";
type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "agentrun-timeout" | "schema-mismatch" | "unsupported-version" | "validation-failed" | "not-found";
type AgentRunRestCompatGroup = "queue" | "sessions" | "aipod-specs" | "aipods" | "runs" | "commands" | "runner";
type AgentRunResourceVerb = "get" | "describe" | "events" | "logs" | "result" | "ack" | "cancel" | "dispatch" | "create" | "apply" | "send" | "explain";