From d44e195ae8d87b98ddf260248ffe4a1ea855372f Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Mon, 22 Jun 2026 03:26:28 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20AgentRun=20unreachable=20?= =?UTF-8?q?=E6=B3=9B=E5=8C=96=E4=B8=8E=20YAML-lane=20GitOps=20=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E5=88=86=E5=8F=89=20(#624)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: expose web-probe diagnostic previews * fix: classify agentrun unreachable root causes --------- Co-authored-by: Codex --- scripts/src/agentrun.ts | 363 +++++++++++++------- scripts/src/platform-infra-observability.ts | 54 ++- 2 files changed, 289 insertions(+), 128 deletions(-) diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index c34c2b16..dff7829d 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -2998,33 +2998,31 @@ async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec: } const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" }); const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image }); - progressEvent("agentrun.yaml-lane.gitops-publish.progress", { - node: spec.nodeId, - lane: spec.lane, - sourceCommit, - status: "submitting", - }); - const gitopsSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishSubmitScript(spec, renderedFiles)]); - const gitopsSubmitPayload = captureJsonPayload(gitopsSubmit); - if (gitopsSubmit.exitCode !== 0 || gitopsSubmitPayload.ok === false) { + const mirrorPrePublish = await runYamlLaneGitMirrorSyncJob(config, spec); + if (mirrorPrePublish.ok !== true) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), - phase: "gitops-publish-submit", + phase: "git-mirror-prepublish-sync", sourceCommit, image, - degradedReason: "yaml-lane-gitops-publish-submit-failed", + degradedReason: "yaml-lane-git-mirror-prepublish-sync-failed", sourceBootstrap: bootstrapPayload, imageBuild: buildPayload, - result: gitopsSubmitPayload, - capture: compactCapture(gitopsSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + result: mirrorPrePublish, valuesPrinted: false, }; } - const gitops = await waitForYamlLaneGitopsPublish(config, spec, sourceCommit, stringOrNull(gitopsSubmitPayload.jobId)); + progressEvent("agentrun.yaml-lane.gitops-publish.progress", { + node: spec.nodeId, + lane: spec.lane, + sourceCommit, + status: "submitting", + }); + const gitops = await runYamlLaneGitopsPublishJob(config, spec, sourceCommit, renderedFiles); const gitopsPayload = gitops.payload; if (gitops.ok !== true || gitopsPayload.ok === false) { return { @@ -3039,26 +3037,27 @@ async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec: degradedReason: "yaml-lane-gitops-publish-failed", sourceBootstrap: bootstrapPayload, imageBuild: buildPayload, - gitopsPublishSubmit: gitopsSubmitPayload, + gitMirrorPrePublish: mirrorPrePublish, result: gitopsPayload, gitopsPublishStatus: gitops, valuesPrinted: false, }; } - const mirror = await runYamlLaneGitMirrorSyncJob(config, spec); - if (mirror.ok !== true) { + const mirrorFlush = await runYamlLaneGitMirrorFlushJob(config, spec); + if (mirrorFlush.ok !== true) { return { ok: false, command: "agentrun control-plane trigger-current", mode: waited ? "confirmed-waited" : "confirmed-trigger", configPath, target: agentRunLaneSummary(spec), - phase: "git-mirror-sync", + phase: "git-mirror-flush", sourceCommit, image, gitops: gitopsPayload, - degradedReason: "yaml-lane-git-mirror-sync-failed", - result: mirror, + gitMirrorPrePublish: mirrorPrePublish, + degradedReason: "yaml-lane-git-mirror-flush-failed", + result: mirrorFlush, valuesPrinted: false, }; } @@ -3078,13 +3077,13 @@ async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec: sourceBootstrap: bootstrapPayload, imageBuildSubmit: buildSubmitPayload, imageBuild: buildPayload, - gitopsPublishSubmit: gitopsSubmitPayload, renderedFiles: { count: renderedFiles.length, digest: renderedFilesDigest(renderedFiles), }, gitops: gitopsPayload, - gitMirror: mirror, + gitMirrorPrePublish: mirrorPrePublish, + gitMirror: mirrorFlush, created: createPayload, capture: compactCapture(created, { full: created.exitCode !== 0, stdoutTailChars: 4000, stderrTailChars: 4000 }), next: { @@ -3853,52 +3852,175 @@ function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string): ].join("\n"); } -function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string { - const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`; +async function runYamlLaneGitopsPublishJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, files: readonly { path: string; content: string }[]): Promise & { ok: boolean; payload: Record }> { + const jobName = `gitops-publish-${spec.nodeId.toLowerCase()}-${spec.lane}-${Date.now().toString(36)}`.slice(0, 63); + const manifest = yamlLaneGitopsPublishJobManifest(spec, files, jobName); + const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); + if (created.exitCode !== 0) { + return { + ok: false, + payload: { ok: false, status: "create-failed", jobName, degradedReason: "gitops-publish-job-create-failed", valuesPrinted: false }, + jobName, + sourceCommit, + phase: "create-job", + capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + const startedAt = Date.now(); + const timeoutMs = 300_000; + let polls = 0; + let lastPayload: Record = {}; + let lastProbe: SshCaptureResult | null = null; + while (Date.now() - startedAt < timeoutMs) { + polls += 1; + lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]); + const probePayload = captureJsonPayload(lastProbe); + const publishPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload); + if (Object.keys(publishPayload).length > 0) lastPayload = publishPayload; + progressEvent("agentrun.yaml-lane.gitops-publish.progress", { + node: spec.nodeId, + lane: spec.lane, + sourceCommit, + jobName, + polls, + status: stringOrNull(publishPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"), + gitopsCommit: stringOrNull(publishPayload.gitopsCommit), + elapsedMs: Date.now() - startedAt, + }); + if (probePayload.succeeded === true) { + if (publishPayload.ok === true && stringOrNull(publishPayload.gitopsCommit) !== null) { + return { ok: true, payload: publishPayload, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false }; + } + return { + ok: false, + payload: { ...publishPayload, ok: false, status: stringOrNull(publishPayload.status) ?? "succeeded-without-result", degradedReason: "gitops-publish-result-missing", jobName, valuesPrinted: false }, + jobName, + polls, + elapsedMs: Date.now() - startedAt, + probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + if (probePayload.failed === true) { + const payload = Object.keys(publishPayload).length > 0 ? publishPayload : { ok: false, status: "failed", degradedReason: "gitops-publish-job-failed", jobName, valuesPrinted: false }; + return { + ok: false, + payload, + jobName, + polls, + elapsedMs: Date.now() - startedAt, + probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; + } + await sleep(5_000); + } + return { + ok: false, + payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-job-timeout", jobName, valuesPrinted: false }, + jobName, + polls, + elapsedMs: Date.now() - startedAt, + probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }), + valuesPrinted: false, + }; +} + +function yamlLaneGitopsPublishPayloadFromProbe(probePayload: Record): Record { + const logsTail = stringOrNull(probePayload.logsTail); + if (logsTail === null) return {}; + const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + for (let index = lines.length - 1; index >= 0; index -= 1) { + const line = lines[index]; + if (!line.startsWith("{") || !line.endsWith("}")) continue; + try { + const parsed = JSON.parse(line) as unknown; + const payload = record(parsed); + if (payload.ok === true || payload.ok === false) return payload; + } catch { + continue; + } + } + return {}; +} + +function yamlLaneGitopsPublishJobManifest(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[], name: string): Record { + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name, + namespace: spec.gitMirror.namespace, + labels: { + "app.kubernetes.io/name": "gitops-publish", + "app.kubernetes.io/part-of": "agentrun", + "agentrun.pikastech.local/lane": spec.version, + "agentrun.pikastech.local/node": spec.nodeId, + "agentrun.pikastech.local/component": "gitops-publish", + }, + }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: 600, + ttlSecondsAfterFinished: 3600, + template: { + metadata: { + labels: { + "app.kubernetes.io/name": "gitops-publish", + "app.kubernetes.io/part-of": "agentrun", + "agentrun.pikastech.local/lane": spec.version, + "agentrun.pikastech.local/node": spec.nodeId, + "agentrun.pikastech.local/component": "gitops-publish", + }, + }, + spec: { + restartPolicy: "Never", + volumes: [yamlLaneGitMirrorCacheVolume(spec)], + containers: [{ + name: "publish", + image: spec.gitMirror.toolsImage, + imagePullPolicy: "IfNotPresent", + command: ["/bin/sh", "-ec", yamlLaneGitopsPublishShell(spec, files, name)], + volumeMounts: [{ name: "cache", mountPath: "/cache" }], + }], + }, + }, + }, + }; +} + +function yamlLaneGitopsPublishShell(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[], jobName: string): string { const filesB64 = Buffer.from(JSON.stringify(files.map((file) => ({ path: file.path, contentBase64: Buffer.from(file.content, "utf8").toString("base64"), }))), "utf8").toString("base64"); return [ "set -eu", - `state_dir=${shQuote(stateDir)}`, - `remote=${shQuote(spec.source.remote)}`, + `repository=${shQuote(spec.source.repository)}`, `gitops_branch=${shQuote(spec.gitops.branch)}`, `gitops_root=${shQuote(spec.deployment.gitopsRoot)}`, `artifact_catalog=${shQuote(spec.deployment.artifactCatalogPath)}`, `files_b64=${shQuote(filesB64)}`, - "mkdir -p \"$state_dir\"", - "job_id=\"gitops-publish-$(date +%s)-$$\"", - "status_file=\"$state_dir/$job_id.json\"", - "stdout_file=\"$state_dir/$job_id.stdout.log\"", - "stderr_file=\"$state_dir/$job_id.stderr.log\"", - "publish_root=\"$state_dir/worktrees\"", - "publish_worktree=\"$publish_root/$job_id\"", - "cat > \"$status_file\" </dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" PUBLISH_WORKSPACE=\"$publish_worktree\" GITOPS_BRANCH=\"$gitops_branch\" node <<'NODE' > \"$status_file\"", - "const code = Number(process.env.CODE || 1);", - "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, publishWorkspace: process.env.PUBLISH_WORKSPACE, gitopsBranch: process.env.GITOPS_BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));", - "NODE", - " fi; exit \"$code\"; }", - " trap write_failed_status EXIT", - " mkdir -p \"$publish_root\"", - " rm -rf \"$publish_worktree\"", - " git clone --no-checkout \"$remote\" \"$publish_worktree\"", - " cd \"$publish_worktree\"", - " git fetch origin \"$gitops_branch\" || true", - " if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then", - " git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"", - " else", - " git checkout --orphan \"$gitops_branch\"", - " git rm -rf . >/dev/null 2>&1 || true", - " fi", - " git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true", - " rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json", - " FILES_B64=\"$files_b64\" node <<'NODE'", + `job_name=${shQuote(jobName)}`, + "remote=\"/cache/${repository}.git\"", + "remote_mode=git-mirror-cache-volume", + "publish_worktree=\"/tmp/$job_name\"", + "write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then printf '%s\\n' \"{\\\"ok\\\":false,\\\"status\\\":\\\"failed\\\",\\\"exitCode\\\":$code,\\\"jobName\\\":\\\"$job_name\\\",\\\"publishWorkspace\\\":\\\"$publish_worktree\\\",\\\"gitopsBranch\\\":\\\"$gitops_branch\\\",\\\"publishRemoteMode\\\":\\\"$remote_mode\\\",\\\"valuesPrinted\\\":false}\"; fi; exit \"$code\"; }", + "trap write_failed_status EXIT", + "rm -rf \"$publish_worktree\"", + "git clone --no-checkout \"$remote\" \"$publish_worktree\"", + "cd \"$publish_worktree\"", + "git fetch origin \"$gitops_branch\" || true", + "if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then", + " git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"", + "else", + " git checkout --orphan \"$gitops_branch\"", + " git rm -rf . >/dev/null 2>&1 || true", + "fi", + "git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true", + "rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json", + "FILES_B64=\"$files_b64\" node <<'NODE'", "const fs = require('node:fs');", "const path = require('node:path');", "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));", @@ -3909,57 +4031,14 @@ function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readon " fs.writeFileSync(target, Buffer.from(file.contentBase64, 'base64'));", "}", "NODE", - " git add source.json \"$artifact_catalog\" \"$gitops_root\"", - " if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi", - " git push -u origin \"$gitops_branch\"", - " gitops_commit=$(git rev-parse HEAD)", - " rm -rf \"$publish_worktree\"", - " CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_ID=\"$job_id\" node <<'NODE' > \"$status_file\"", - "console.log(JSON.stringify({ ok: true, status: 'succeeded', jobId: process.env.JOB_ID, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));", + "git add source.json \"$artifact_catalog\" \"$gitops_root\"", + "if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi", + "git push -u origin \"$gitops_branch\"", + "gitops_commit=$(git rev-parse HEAD)", + "CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_NAME=\"$job_name\" PUBLISH_REMOTE_MODE=\"$remote_mode\" node <<'NODE'", + "console.log(JSON.stringify({ ok: true, status: 'succeeded', jobName: process.env.JOB_NAME, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, publishRemoteMode: process.env.PUBLISH_REMOTE_MODE || null, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));", "NODE", - " trap - EXIT", - ") >\"$stdout_file\" 2>\"$stderr_file\" &", - "pid=$!", - "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'", - "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));", - "NODE", - ].join("\n"); -} - -async function waitForYamlLaneGitopsPublish(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise & { ok: boolean; payload: Record }> { - if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "gitops-publish-job-id-missing", valuesPrinted: false } }; - const startedAt = Date.now(); - const timeoutMs = 300_000; - let lastPayload: Record = {}; - let polls = 0; - while (Date.now() - startedAt < timeoutMs) { - polls += 1; - const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishStatusScript(spec, jobId)]); - const payload = captureJsonPayload(probe); - lastPayload = payload; - progressEvent("agentrun.yaml-lane.gitops-publish.progress", { - node: spec.nodeId, - lane: spec.lane, - sourceCommit, - jobId, - polls, - status: stringOrNull(payload.status) ?? "unknown", - gitopsCommit: stringOrNull(payload.gitopsCommit), - elapsedMs: Date.now() - startedAt, - }); - if (payload.ok === true && stringOrNull(payload.gitopsCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt }; - if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt }; - await sleep(5_000); - } - return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt }; -} - -function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string): string { - const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`; - return [ - "set +e", - `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`, - "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi", + "trap - EXIT", ].join("\n"); } @@ -4028,14 +4107,23 @@ function agentRunGitMirrorRetrySummary(attempts: Record[]): Rec } async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise> { + return await runYamlLaneGitMirrorJob(config, spec, "sync"); +} + +async function runYamlLaneGitMirrorFlushJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise> { + return await runYamlLaneGitMirrorJob(config, spec, "flush"); +} + +async function runYamlLaneGitMirrorJob(config: UniDeskConfig, spec: AgentRunLaneSpec, action: "sync" | "flush"): Promise> { const attempts: Record[] = []; for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) { - const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63); - const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName); + const jobPrefix = action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix; + const jobName = `${jobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63); + const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName); const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); if (created.exitCode !== 0) { const result = { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false }; - const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result); + const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result); attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure); @@ -4054,6 +4142,7 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun progressEvent("agentrun.yaml-lane.git-mirror.progress", { node: spec.nodeId, lane: spec.lane, + action, jobName, polls, retryAttempt: attempt, @@ -4069,7 +4158,7 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun } if (payload.failed === true) { const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; - const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result); + const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result); attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure); @@ -4081,9 +4170,10 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun await sleep(5_000); } if (attempts.length > 0 && record(attempts[attempts.length - 1]).jobName === jobName) continue; - const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; - const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result); - attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); + const degradedReason = `git-mirror-${action}-job-timeout`; + const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason, capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; + const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result); + attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure); await sleep(agentRunGitMirrorRetryDelayMs(attempt)); @@ -4091,7 +4181,7 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun } return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) }; } - return { ok: false, degradedReason: "git-mirror-sync-retry-loop-exhausted", retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false }; + return { ok: false, degradedReason: `git-mirror-${action}-retry-loop-exhausted`, retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false }; } function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCommit: string, pipelineRun: string): string { @@ -6000,7 +6090,8 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod, } response = await fetch(new URL(pathValue, clientConfig.manager.baseUrl), init); } catch (error) { - throw new AgentRunRestError("agentrun-unreachable", `AgentRun server is unreachable for ${method} ${pathValue}: ${error instanceof Error ? error.message : String(error)}`, { bridge: { ...bridgeBase, elapsedMs: Date.now() - startedAt } }); + const timedOut = isAbortLikeError(error); + throw new AgentRunRestError(timedOut ? "agentrun-timeout" : "agentrun-unreachable", timedOut ? `AgentRun server request timed out for ${method} ${pathValue} after ${clientConfig.manager.timeoutMs}ms` : `AgentRun server is unreachable for ${method} ${pathValue}: ${error instanceof Error ? error.message : String(error)}`, { bridge: { ...bridgeBase, elapsedMs: Date.now() - startedAt } }); } finally { clearTimeout(timeout); } @@ -6032,14 +6123,21 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod, }; } +function isAbortLikeError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + return error.name === "AbortError" || /abort|timed out|timeout/iu.test(error.message); +} + async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body: unknown, clientConfig: AgentRunClientConfig, target: AgentRunRestTarget): Promise> { const bridgeBase = agentRunLaneRestBridgeMetadata(clientConfig, target, method, pathValue); const startedAt = Date.now(); + const proxyBaseUrl = `http://127.0.0.1:${target.spec.runtime.managerPort}`; const request = { method, path: pathValue, body: body ?? null, - baseUrl: target.spec.runtime.internalBaseUrl, + baseUrl: proxyBaseUrl, + serviceBaseUrl: target.spec.runtime.internalBaseUrl, timeoutMs: clientConfig.manager.timeoutMs, authEnv: clientConfig.auth.env, header: clientConfig.auth.header, @@ -6054,7 +6152,9 @@ async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMeth } const proxy = captureJsonPayload(captureResult); if (proxy.ok !== true) { - throw new AgentRunRestError("agentrun-unreachable", stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "valuesPrinted"]) }); + const proxyFailureKind = stringOrNull(proxy.failureKind); + const failureKind: AgentRunFailureKind = proxyFailureKind === "manager-pod-fetch-timeout" ? "agentrun-timeout" : "agentrun-unreachable"; + throw new AgentRunRestError(failureKind, stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "path", "baseUrl", "valuesPrinted"]) }); } const httpStatus = nonNegativeIntegerOrNull(proxy.httpStatus) ?? 0; const text = stringOrNull(proxy.text) ?? ""; @@ -6089,8 +6189,17 @@ function agentRunLaneRestProxyScript(spec: AgentRunLaneSpec, request: Record controller.abort(), timeoutMs); let response; try { - response = await fetch(new URL(String(input.path || "/"), String(input.baseUrl || "http://127.0.0.1:8080")).toString(), { - method: String(input.method || "GET"), + response = await fetch(new URL(path, baseUrl).toString(), { + method, headers, body, signal: controller.signal, @@ -6117,7 +6225,10 @@ try { const text = await response.text(); console.log(JSON.stringify({ ok: true, httpStatus: response.status, statusText: response.statusText, text, elapsedMs: Date.now() - startedAt, valuesPrinted: false })); } catch (error) { - console.log(JSON.stringify({ ok: false, failureKind: "manager-pod-fetch-failed", message: error instanceof Error ? error.message : String(error), elapsedMs: Date.now() - startedAt, valuesPrinted: false })); + const name = error instanceof Error ? error.name : ""; + const message = error instanceof Error ? error.message : String(error); + const timedOut = name === "AbortError" || /abort|timed out|timeout/iu.test(message); + console.log(JSON.stringify({ ok: false, failureKind: timedOut ? "manager-pod-fetch-timeout" : "manager-pod-fetch-failed", message: timedOut ? "AgentRun manager request timed out for " + method + " " + path + " after " + timeoutMs + "ms" : message, elapsedMs: Date.now() - startedAt, path, baseUrl, valuesPrinted: false })); } `; return [ @@ -6733,7 +6844,7 @@ function safeAgentRunEnvelope(envelope: Record): Record [key, joinValues(identity[key], 88)]) + .filter((row) => row[1] !== "-"); const next = asPlainRecord(input.result.next); const lines = [ `platform-infra observability trace (${input.ok ? "ok" : "not-ok"})`, @@ -835,6 +839,9 @@ function renderTraceTable(input: { joinValues(input.result.businessTraceIds, 34), ]]), "", + "Identity:", + formatTable(["FIELD", "VALUE"], identityRows.length > 0 ? identityRows : [["-", "-"]]), + "", "Key spans:", formatTable(["NAME", "SERVICE", "DETAIL"], spanRows.length > 0 ? spanRows : [["-", "-", "-"]]), "", @@ -2125,6 +2132,7 @@ if not isinstance(parsed, dict): spans = [] services = set() business_trace_ids = set() +identity_values = collections.defaultdict(set) name_counts = collections.Counter() error_spans = [] matched_spans = [] @@ -2146,6 +2154,11 @@ for batch in batches_from_body(parsed): attrs = item.get("attributes", {}) if isinstance(attrs.get("traceId"), str): business_trace_ids.add(attrs.get("traceId")) + if isinstance(attrs, dict): + for key in ("runId", "commandId", "runnerJobId", "runnerId", "backendProfile", "sourceCommit", "jobName", "podName"): + value = attrs.get(key) + if value not in (None, ""): + identity_values[key].add(str(value)) name = str(item.get("name") or "") name_counts[name] += 1 spans.append(item) @@ -2172,6 +2185,7 @@ payload = { "serviceCount": len(services), "services": sorted(services), "businessTraceIds": sorted(business_trace_ids)[:20], + "identity": {key: sorted(values)[:8] for key, values in identity_values.items() if values}, "errorSpanCount": len(error_spans), "matchedSpanCount": len(matched_spans) if GREP is not None else None, "grep": GREP, @@ -3009,6 +3023,9 @@ def http_status_summary(spans): def agentrun_summary(spans): terminal_spans = [] latest_seq = None + failure_kind = None + failure_message = None + terminal_category = None seq_keys = ("maxSeq", "traceLastSeq", "endSeq", "seq", "eventSeq", "sourceSeq", "sourceLatestSeq", "latestSeq", "lastSeq") for item in spans: service = str(item.get("service") or "") @@ -3024,9 +3041,17 @@ def agentrun_summary(spans): derived = terminal_status if not derived and name.startswith("runner_terminal."): derived = name.split(".", 1)[1] + if attrs.get("failureKind") not in (None, ""): + failure_kind = attrs.get("failureKind") + if attrs.get("failureMessage") not in (None, ""): + failure_message = attrs.get("failureMessage") + if attrs.get("terminalCategory") not in (None, ""): + terminal_category = attrs.get("terminalCategory") terminal_spans.append({ "status": derived, "eventType": attrs.get("eventType"), + "failureKind": attrs.get("failureKind"), + "terminalCategory": attrs.get("terminalCategory"), "span": public_span(item), }) terminal_status = terminal_spans[-1]["status"] if terminal_spans else None @@ -3047,6 +3072,9 @@ def agentrun_summary(spans): runner_classification = "unknown" return { "terminalStatus": terminal_status, + "failureKind": failure_kind, + "failureMessage": failure_message, + "terminalCategory": terminal_category, "latestSeq": latest_seq, "terminalEventType": terminal_event_type, "terminalSpans": terminal_spans, @@ -3176,6 +3204,19 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error "summary": "HWLAB projection stale is suspected because AgentRun is terminal but HWLAB read-model evidence is incomplete.", "evidence": lag_summary.get("reasons", []), }) + if agentrun.get("terminalStatus") in ("failed", "error", "timeout", "blocked", "cancelled"): + candidates.append({ + "code": "agentrun_terminal_failed", + "label": "AgentRun terminal failed", + "confidence": 0.9, + "summary": "AgentRun reached a non-success terminal state; inspect result/events/runnerjob before retrying or continuing the session.", + "evidence": { + "terminalStatus": agentrun.get("terminalStatus"), + "failureKind": agentrun.get("failureKind"), + "terminalCategory": agentrun.get("terminalCategory"), + "runnerProviderClassification": agentrun.get("runnerProviderClassification"), + }, + }) if agentrun.get("terminalStatus") == "completed": candidates.append({ "code": "agentrun_completed_not_provider_blocked", @@ -3451,11 +3492,18 @@ service_path["complete"] = all(service in services for service in expected_servi facts = [] if http_summary.get("actorForbidden"): facts.append("actor forbidden") -if agentrun.get("terminalStatus") == "completed": +terminal_status = agentrun.get("terminalStatus") +if terminal_status in ("failed", "error", "timeout", "blocked", "cancelled"): + failure_kind = agentrun.get("failureKind") + if failure_kind: + facts.append(f"AgentRun terminal failed ({failure_kind})") + else: + facts.append("AgentRun terminal failed") +if terminal_status == "completed": facts.append("AgentRun completed") if lag.get("status") in ("confirmed", "suspected"): facts.append("HWLAB projection stale") -if idle_warning_spans and agentrun.get("terminalStatus") in (None, ""): +if idle_warning_spans and terminal_status in (None, ""): facts.append("AgentRun runner idle warnings active") if not facts: facts.append("no dominant Code Agent root cause classified") @@ -3466,6 +3514,8 @@ summary = { "projectionLag": lag.get("status"), "runnerProvider": agentrun.get("runnerProviderClassification"), "actorForbidden": http_summary.get("actorForbidden"), + "terminalStatus": terminal_status, + "failureKind": agentrun.get("failureKind"), }, } evidence = {