修复 AgentRun unreachable 泛化与 YAML-lane GitOps 发布分叉 (#624)

* fix: expose web-probe diagnostic previews

* fix: classify agentrun unreachable root causes

---------

Co-authored-by: Codex <codex@noreply.local>

scripts/src/agentrun.ts

@@ -2998,33 +2998,31 @@ async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec:
   }
   const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" });
   const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image });
-  progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
-    node: spec.nodeId,
-    lane: spec.lane,
-    sourceCommit,
-    status: "submitting",
-  });
-  const gitopsSubmit = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishSubmitScript(spec, renderedFiles)]);
-  const gitopsSubmitPayload = captureJsonPayload(gitopsSubmit);
-  if (gitopsSubmit.exitCode !== 0 || gitopsSubmitPayload.ok === false) {
+  const mirrorPrePublish = await runYamlLaneGitMirrorSyncJob(config, spec);
+  if (mirrorPrePublish.ok !== true) {
     return {
       ok: false,
       command: "agentrun control-plane trigger-current",
       mode: waited ? "confirmed-waited" : "confirmed-trigger",
       configPath,
       target: agentRunLaneSummary(spec),
-      phase: "gitops-publish-submit",
+      phase: "git-mirror-prepublish-sync",
       sourceCommit,
       image,
-      degradedReason: "yaml-lane-gitops-publish-submit-failed",
+      degradedReason: "yaml-lane-git-mirror-prepublish-sync-failed",
       sourceBootstrap: bootstrapPayload,
       imageBuild: buildPayload,
-      result: gitopsSubmitPayload,
-      capture: compactCapture(gitopsSubmit, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
+      result: mirrorPrePublish,
       valuesPrinted: false,
     };
   }
-  const gitops = await waitForYamlLaneGitopsPublish(config, spec, sourceCommit, stringOrNull(gitopsSubmitPayload.jobId));
+  progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
+    node: spec.nodeId,
+    lane: spec.lane,
+    sourceCommit,
+    status: "submitting",
+  });
+  const gitops = await runYamlLaneGitopsPublishJob(config, spec, sourceCommit, renderedFiles);
   const gitopsPayload = gitops.payload;
   if (gitops.ok !== true || gitopsPayload.ok === false) {
     return {
@@ -3039,26 +3037,27 @@ async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec:
       degradedReason: "yaml-lane-gitops-publish-failed",
       sourceBootstrap: bootstrapPayload,
       imageBuild: buildPayload,
-      gitopsPublishSubmit: gitopsSubmitPayload,
+      gitMirrorPrePublish: mirrorPrePublish,
       result: gitopsPayload,
       gitopsPublishStatus: gitops,
       valuesPrinted: false,
     };
   }
-  const mirror = await runYamlLaneGitMirrorSyncJob(config, spec);
-  if (mirror.ok !== true) {
+  const mirrorFlush = await runYamlLaneGitMirrorFlushJob(config, spec);
+  if (mirrorFlush.ok !== true) {
     return {
       ok: false,
       command: "agentrun control-plane trigger-current",
       mode: waited ? "confirmed-waited" : "confirmed-trigger",
       configPath,
       target: agentRunLaneSummary(spec),
-      phase: "git-mirror-sync",
+      phase: "git-mirror-flush",
       sourceCommit,
       image,
       gitops: gitopsPayload,
-      degradedReason: "yaml-lane-git-mirror-sync-failed",
-      result: mirror,
+      gitMirrorPrePublish: mirrorPrePublish,
+      degradedReason: "yaml-lane-git-mirror-flush-failed",
+      result: mirrorFlush,
       valuesPrinted: false,
     };
   }
@@ -3078,13 +3077,13 @@ async function triggerCurrentYamlLaneConfirmedSteps(config: UniDeskConfig, spec:
     sourceBootstrap: bootstrapPayload,
     imageBuildSubmit: buildSubmitPayload,
     imageBuild: buildPayload,
-    gitopsPublishSubmit: gitopsSubmitPayload,
     renderedFiles: {
       count: renderedFiles.length,
       digest: renderedFilesDigest(renderedFiles),
     },
     gitops: gitopsPayload,
-    gitMirror: mirror,
+    gitMirrorPrePublish: mirrorPrePublish,
+    gitMirror: mirrorFlush,
     created: createPayload,
     capture: compactCapture(created, { full: created.exitCode !== 0, stdoutTailChars: 4000, stderrTailChars: 4000 }),
     next: {
@@ -3853,52 +3852,175 @@ function yamlLaneBuildImageStatusScript(spec: AgentRunLaneSpec, jobId: string):
   ].join("\n");
 }
 
-function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[]): string {
-  const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
+async function runYamlLaneGitopsPublishJob(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, files: readonly { path: string; content: string }[]): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
+  const jobName = `gitops-publish-${spec.nodeId.toLowerCase()}-${spec.lane}-${Date.now().toString(36)}`.slice(0, 63);
+  const manifest = yamlLaneGitopsPublishJobManifest(spec, files, jobName);
+  const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
+  if (created.exitCode !== 0) {
+    return {
+      ok: false,
+      payload: { ok: false, status: "create-failed", jobName, degradedReason: "gitops-publish-job-create-failed", valuesPrinted: false },
+      jobName,
+      sourceCommit,
+      phase: "create-job",
+      capture: compactCapture(created, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
+      valuesPrinted: false,
+    };
+  }
+  const startedAt = Date.now();
+  const timeoutMs = 300_000;
+  let polls = 0;
+  let lastPayload: Record<string, unknown> = {};
+  let lastProbe: SshCaptureResult | null = null;
+  while (Date.now() - startedAt < timeoutMs) {
+    polls += 1;
+    lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]);
+    const probePayload = captureJsonPayload(lastProbe);
+    const publishPayload = yamlLaneGitopsPublishPayloadFromProbe(probePayload);
+    if (Object.keys(publishPayload).length > 0) lastPayload = publishPayload;
+    progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
+      node: spec.nodeId,
+      lane: spec.lane,
+      sourceCommit,
+      jobName,
+      polls,
+      status: stringOrNull(publishPayload.status) ?? (probePayload.succeeded === true ? "succeeded" : probePayload.failed === true ? "failed" : "running"),
+      gitopsCommit: stringOrNull(publishPayload.gitopsCommit),
+      elapsedMs: Date.now() - startedAt,
+    });
+    if (probePayload.succeeded === true) {
+      if (publishPayload.ok === true && stringOrNull(publishPayload.gitopsCommit) !== null) {
+        return { ok: true, payload: publishPayload, jobName, polls, elapsedMs: Date.now() - startedAt, probe: compactCapture(lastProbe), valuesPrinted: false };
+      }
+      return {
+        ok: false,
+        payload: { ...publishPayload, ok: false, status: stringOrNull(publishPayload.status) ?? "succeeded-without-result", degradedReason: "gitops-publish-result-missing", jobName, valuesPrinted: false },
+        jobName,
+        polls,
+        elapsedMs: Date.now() - startedAt,
+        probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
+        valuesPrinted: false,
+      };
+    }
+    if (probePayload.failed === true) {
+      const payload = Object.keys(publishPayload).length > 0 ? publishPayload : { ok: false, status: "failed", degradedReason: "gitops-publish-job-failed", jobName, valuesPrinted: false };
+      return {
+        ok: false,
+        payload,
+        jobName,
+        polls,
+        elapsedMs: Date.now() - startedAt,
+        probe: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
+        valuesPrinted: false,
+      };
+    }
+    await sleep(5_000);
+  }
+  return {
+    ok: false,
+    payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-job-timeout", jobName, valuesPrinted: false },
+    jobName,
+    polls,
+    elapsedMs: Date.now() - startedAt,
+    probe: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 5000 }),
+    valuesPrinted: false,
+  };
+}
+
+function yamlLaneGitopsPublishPayloadFromProbe(probePayload: Record<string, unknown>): Record<string, unknown> {
+  const logsTail = stringOrNull(probePayload.logsTail);
+  if (logsTail === null) return {};
+  const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
+  for (let index = lines.length - 1; index >= 0; index -= 1) {
+    const line = lines[index];
+    if (!line.startsWith("{") || !line.endsWith("}")) continue;
+    try {
+      const parsed = JSON.parse(line) as unknown;
+      const payload = record(parsed);
+      if (payload.ok === true || payload.ok === false) return payload;
+    } catch {
+      continue;
+    }
+  }
+  return {};
+}
+
+function yamlLaneGitopsPublishJobManifest(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[], name: string): Record<string, unknown> {
+  return {
+    apiVersion: "batch/v1",
+    kind: "Job",
+    metadata: {
+      name,
+      namespace: spec.gitMirror.namespace,
+      labels: {
+        "app.kubernetes.io/name": "gitops-publish",
+        "app.kubernetes.io/part-of": "agentrun",
+        "agentrun.pikastech.local/lane": spec.version,
+        "agentrun.pikastech.local/node": spec.nodeId,
+        "agentrun.pikastech.local/component": "gitops-publish",
+      },
+    },
+    spec: {
+      backoffLimit: 0,
+      activeDeadlineSeconds: 600,
+      ttlSecondsAfterFinished: 3600,
+      template: {
+        metadata: {
+          labels: {
+            "app.kubernetes.io/name": "gitops-publish",
+            "app.kubernetes.io/part-of": "agentrun",
+            "agentrun.pikastech.local/lane": spec.version,
+            "agentrun.pikastech.local/node": spec.nodeId,
+            "agentrun.pikastech.local/component": "gitops-publish",
+          },
+        },
+        spec: {
+          restartPolicy: "Never",
+          volumes: [yamlLaneGitMirrorCacheVolume(spec)],
+          containers: [{
+            name: "publish",
+            image: spec.gitMirror.toolsImage,
+            imagePullPolicy: "IfNotPresent",
+            command: ["/bin/sh", "-ec", yamlLaneGitopsPublishShell(spec, files, name)],
+            volumeMounts: [{ name: "cache", mountPath: "/cache" }],
+          }],
+        },
+      },
+    },
+  };
+}
+
+function yamlLaneGitopsPublishShell(spec: AgentRunLaneSpec, files: readonly { path: string; content: string }[], jobName: string): string {
   const filesB64 = Buffer.from(JSON.stringify(files.map((file) => ({
     path: file.path,
     contentBase64: Buffer.from(file.content, "utf8").toString("base64"),
   }))), "utf8").toString("base64");
   return [
     "set -eu",
-    `state_dir=${shQuote(stateDir)}`,
-    `remote=${shQuote(spec.source.remote)}`,
+    `repository=${shQuote(spec.source.repository)}`,
     `gitops_branch=${shQuote(spec.gitops.branch)}`,
     `gitops_root=${shQuote(spec.deployment.gitopsRoot)}`,
     `artifact_catalog=${shQuote(spec.deployment.artifactCatalogPath)}`,
     `files_b64=${shQuote(filesB64)}`,
-    "mkdir -p \"$state_dir\"",
-    "job_id=\"gitops-publish-$(date +%s)-$$\"",
-    "status_file=\"$state_dir/$job_id.json\"",
-    "stdout_file=\"$state_dir/$job_id.stdout.log\"",
-    "stderr_file=\"$state_dir/$job_id.stderr.log\"",
-    "publish_root=\"$state_dir/worktrees\"",
-    "publish_worktree=\"$publish_root/$job_id\"",
-    "cat > \"$status_file\" <<EOF",
-    "{\"ok\":false,\"status\":\"running\",\"jobId\":\"$job_id\",\"publishWorkspace\":\"$publish_worktree\",\"gitopsBranch\":\"$gitops_branch\",\"valuesPrinted\":false}",
-    "EOF",
-    "(",
-    "  set -eu",
-    "  write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then tail_text=$(tail -n 80 \"$stderr_file\" 2>/dev/null | sed 's/\"/\\\\\"/g' | tr '\\n' ' ' | cut -c1-4000); CODE=\"$code\" ERROR_TAIL=\"$tail_text\" JOB_ID=\"$job_id\" PUBLISH_WORKSPACE=\"$publish_worktree\" GITOPS_BRANCH=\"$gitops_branch\" node <<'NODE' > \"$status_file\"",
-    "const code = Number(process.env.CODE || 1);",
-    "console.log(JSON.stringify({ ok: false, status: 'failed', exitCode: code, jobId: process.env.JOB_ID, publishWorkspace: process.env.PUBLISH_WORKSPACE, gitopsBranch: process.env.GITOPS_BRANCH, errorTail: process.env.ERROR_TAIL || null, valuesPrinted: false }));",
-    "NODE",
-    "  fi; exit \"$code\"; }",
-    "  trap write_failed_status EXIT",
-    "  mkdir -p \"$publish_root\"",
-    "  rm -rf \"$publish_worktree\"",
-    "  git clone --no-checkout \"$remote\" \"$publish_worktree\"",
-    "  cd \"$publish_worktree\"",
-    "  git fetch origin \"$gitops_branch\" || true",
-    "  if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then",
-    "    git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"",
-    "  else",
-    "    git checkout --orphan \"$gitops_branch\"",
-    "    git rm -rf . >/dev/null 2>&1 || true",
-    "  fi",
-    "  git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true",
-    "  rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json",
-    "  FILES_B64=\"$files_b64\" node <<'NODE'",
+    `job_name=${shQuote(jobName)}`,
+    "remote=\"/cache/${repository}.git\"",
+    "remote_mode=git-mirror-cache-volume",
+    "publish_worktree=\"/tmp/$job_name\"",
+    "write_failed_status() { code=$?; if [ \"$code\" -ne 0 ]; then printf '%s\\n' \"{\\\"ok\\\":false,\\\"status\\\":\\\"failed\\\",\\\"exitCode\\\":$code,\\\"jobName\\\":\\\"$job_name\\\",\\\"publishWorkspace\\\":\\\"$publish_worktree\\\",\\\"gitopsBranch\\\":\\\"$gitops_branch\\\",\\\"publishRemoteMode\\\":\\\"$remote_mode\\\",\\\"valuesPrinted\\\":false}\"; fi; exit \"$code\"; }",
+    "trap write_failed_status EXIT",
+    "rm -rf \"$publish_worktree\"",
+    "git clone --no-checkout \"$remote\" \"$publish_worktree\"",
+    "cd \"$publish_worktree\"",
+    "git fetch origin \"$gitops_branch\" || true",
+    "if git rev-parse --verify \"refs/remotes/origin/$gitops_branch^{commit}\" >/dev/null 2>&1; then",
+    "  git checkout -B \"$gitops_branch\" \"refs/remotes/origin/$gitops_branch\"",
+    "else",
+    "  git checkout --orphan \"$gitops_branch\"",
+    "  git rm -rf . >/dev/null 2>&1 || true",
+    "fi",
+    "git rm -rf --ignore-unmatch \"$gitops_root\" \"$artifact_catalog\" source.json >/dev/null 2>&1 || true",
+    "rm -rf \"$gitops_root\" \"$artifact_catalog\" source.json",
+    "FILES_B64=\"$files_b64\" node <<'NODE'",
     "const fs = require('node:fs');",
     "const path = require('node:path');",
     "const files = JSON.parse(Buffer.from(process.env.FILES_B64 || '', 'base64').toString('utf8'));",
@@ -3909,57 +4031,14 @@ function yamlLaneGitopsPublishSubmitScript(spec: AgentRunLaneSpec, files: readon
     "  fs.writeFileSync(target, Buffer.from(file.contentBase64, 'base64'));",
     "}",
     "NODE",
-    "  git add source.json \"$artifact_catalog\" \"$gitops_root\"",
-    "  if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi",
-    "  git push -u origin \"$gitops_branch\"",
-    "  gitops_commit=$(git rev-parse HEAD)",
-    "  rm -rf \"$publish_worktree\"",
-    "  CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_ID=\"$job_id\" node <<'NODE' > \"$status_file\"",
-    "console.log(JSON.stringify({ ok: true, status: 'succeeded', jobId: process.env.JOB_ID, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));",
+    "git add source.json \"$artifact_catalog\" \"$gitops_root\"",
+    "if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=agentrun@unidesk.local -c user.name='UniDesk AgentRun Ops' commit -m \"deploy: render AgentRun ${gitops_branch} from UniDesk YAML\"; fi",
+    "git push -u origin \"$gitops_branch\"",
+    "gitops_commit=$(git rev-parse HEAD)",
+    "CHANGED=\"$changed\" GITOPS_BRANCH=\"$gitops_branch\" GITOPS_COMMIT=\"$gitops_commit\" FILE_COUNT=\"" + String(files.length) + "\" JOB_NAME=\"$job_name\" PUBLISH_REMOTE_MODE=\"$remote_mode\" node <<'NODE'",
+    "console.log(JSON.stringify({ ok: true, status: 'succeeded', jobName: process.env.JOB_NAME, changed: process.env.CHANGED === 'true', gitopsBranch: process.env.GITOPS_BRANCH, gitopsCommit: process.env.GITOPS_COMMIT, publishRemoteMode: process.env.PUBLISH_REMOTE_MODE || null, fileCount: Number(process.env.FILE_COUNT || 0), valuesPrinted: false }));",
     "NODE",
-    "  trap - EXIT",
-    ") >\"$stdout_file\" 2>\"$stderr_file\" &",
-    "pid=$!",
-    "JOB_PID=\"$pid\" JOB_ID=\"$job_id\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" node <<'NODE'",
-    "console.log(JSON.stringify({ ok: true, status: 'submitted', jobId: process.env.JOB_ID, pid: Number(process.env.JOB_PID), statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, valuesPrinted: false }));",
-    "NODE",
-  ].join("\n");
-}
-
-async function waitForYamlLaneGitopsPublish(config: UniDeskConfig, spec: AgentRunLaneSpec, sourceCommit: string, jobId: string | null): Promise<Record<string, unknown> & { ok: boolean; payload: Record<string, unknown> }> {
-  if (jobId === null) return { ok: false, payload: { ok: false, degradedReason: "gitops-publish-job-id-missing", valuesPrinted: false } };
-  const startedAt = Date.now();
-  const timeoutMs = 300_000;
-  let lastPayload: Record<string, unknown> = {};
-  let polls = 0;
-  while (Date.now() - startedAt < timeoutMs) {
-    polls += 1;
-    const probe = await capture(config, spec.nodeRoute, ["sh", "--", yamlLaneGitopsPublishStatusScript(spec, jobId)]);
-    const payload = captureJsonPayload(probe);
-    lastPayload = payload;
-    progressEvent("agentrun.yaml-lane.gitops-publish.progress", {
-      node: spec.nodeId,
-      lane: spec.lane,
-      sourceCommit,
-      jobId,
-      polls,
-      status: stringOrNull(payload.status) ?? "unknown",
-      gitopsCommit: stringOrNull(payload.gitopsCommit),
-      elapsedMs: Date.now() - startedAt,
-    });
-    if (payload.ok === true && stringOrNull(payload.gitopsCommit) !== null) return { ok: true, payload, polls, elapsedMs: Date.now() - startedAt };
-    if (payload.status === "failed") return { ok: false, payload, polls, elapsedMs: Date.now() - startedAt };
-    await sleep(5_000);
-  }
-  return { ok: false, payload: { ...lastPayload, ok: false, status: "timeout", degradedReason: "gitops-publish-timeout", valuesPrinted: false }, polls, elapsedMs: Date.now() - startedAt };
-}
-
-function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string): string {
-  const stateDir = `/tmp/unidesk-agentrun-gitops-${spec.nodeId}-${spec.lane}`;
-  return [
-    "set +e",
-    `status_file=${shQuote(`${stateDir}/${jobId}.json`)}`,
-    "if [ -f \"$status_file\" ]; then cat \"$status_file\"; else printf '{\"ok\":false,\"status\":\"missing\",\"valuesPrinted\":false}\\n'; fi",
+    "trap - EXIT",
   ].join("\n");
 }
 
@@ -4028,14 +4107,23 @@ function agentRunGitMirrorRetrySummary(attempts: Record<string, unknown>[]): Rec
 }
 
 async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise<Record<string, unknown>> {
+  return await runYamlLaneGitMirrorJob(config, spec, "sync");
+}
+
+async function runYamlLaneGitMirrorFlushJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise<Record<string, unknown>> {
+  return await runYamlLaneGitMirrorJob(config, spec, "flush");
+}
+
+async function runYamlLaneGitMirrorJob(config: UniDeskConfig, spec: AgentRunLaneSpec, action: "sync" | "flush"): Promise<Record<string, unknown>> {
   const attempts: Record<string, unknown>[] = [];
   for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) {
-    const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63);
-    const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName);
+    const jobPrefix = action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix;
+    const jobName = `${jobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63);
+    const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName);
     const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
     if (created.exitCode !== 0) {
       const result = { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false };
-      const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
+      const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result);
       attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
       if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
         progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
@@ -4054,6 +4142,7 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun
       progressEvent("agentrun.yaml-lane.git-mirror.progress", {
         node: spec.nodeId,
         lane: spec.lane,
+        action,
         jobName,
         polls,
         retryAttempt: attempt,
@@ -4069,7 +4158,7 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun
       }
       if (payload.failed === true) {
         const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
-        const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
+        const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result);
         attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
         if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
           progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
@@ -4081,9 +4170,10 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun
       await sleep(5_000);
     }
     if (attempts.length > 0 && record(attempts[attempts.length - 1]).jobName === jobName) continue;
-    const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
-    const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
-    attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
+    const degradedReason = `git-mirror-${action}-job-timeout`;
+    const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason, capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
+    const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result);
+    attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
     if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
       progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
       await sleep(agentRunGitMirrorRetryDelayMs(attempt));
@@ -4091,7 +4181,7 @@ async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRun
     }
     return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) };
   }
-  return { ok: false, degradedReason: "git-mirror-sync-retry-loop-exhausted", retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false };
+  return { ok: false, degradedReason: `git-mirror-${action}-retry-loop-exhausted`, retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false };
 }
 
 function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCommit: string, pipelineRun: string): string {
@@ -6000,7 +6090,8 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
     }
     response = await fetch(new URL(pathValue, clientConfig.manager.baseUrl), init);
   } catch (error) {
-    throw new AgentRunRestError("agentrun-unreachable", `AgentRun server is unreachable for ${method} ${pathValue}: ${error instanceof Error ? error.message : String(error)}`, { bridge: { ...bridgeBase, elapsedMs: Date.now() - startedAt } });
+    const timedOut = isAbortLikeError(error);
+    throw new AgentRunRestError(timedOut ? "agentrun-timeout" : "agentrun-unreachable", timedOut ? `AgentRun server request timed out for ${method} ${pathValue} after ${clientConfig.manager.timeoutMs}ms` : `AgentRun server is unreachable for ${method} ${pathValue}: ${error instanceof Error ? error.message : String(error)}`, { bridge: { ...bridgeBase, elapsedMs: Date.now() - startedAt } });
   } finally {
     clearTimeout(timeout);
   }
@@ -6032,14 +6123,21 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
   };
 }
 
+function isAbortLikeError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+  return error.name === "AbortError" || /abort|timed out|timeout/iu.test(error.message);
+}
+
 async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body: unknown, clientConfig: AgentRunClientConfig, target: AgentRunRestTarget): Promise<Record<string, unknown>> {
   const bridgeBase = agentRunLaneRestBridgeMetadata(clientConfig, target, method, pathValue);
   const startedAt = Date.now();
+  const proxyBaseUrl = `http://127.0.0.1:${target.spec.runtime.managerPort}`;
   const request = {
     method,
     path: pathValue,
     body: body ?? null,
-    baseUrl: target.spec.runtime.internalBaseUrl,
+    baseUrl: proxyBaseUrl,
+    serviceBaseUrl: target.spec.runtime.internalBaseUrl,
     timeoutMs: clientConfig.manager.timeoutMs,
     authEnv: clientConfig.auth.env,
     header: clientConfig.auth.header,
@@ -6054,7 +6152,9 @@ async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMeth
   }
   const proxy = captureJsonPayload(captureResult);
   if (proxy.ok !== true) {
-    throw new AgentRunRestError("agentrun-unreachable", stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "valuesPrinted"]) });
+    const proxyFailureKind = stringOrNull(proxy.failureKind);
+    const failureKind: AgentRunFailureKind = proxyFailureKind === "manager-pod-fetch-timeout" ? "agentrun-timeout" : "agentrun-unreachable";
+    throw new AgentRunRestError(failureKind, stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "path", "baseUrl", "valuesPrinted"]) });
   }
   const httpStatus = nonNegativeIntegerOrNull(proxy.httpStatus) ?? 0;
   const text = stringOrNull(proxy.text) ?? "";
@@ -6089,8 +6189,17 @@ function agentRunLaneRestProxyScript(spec: AgentRunLaneSpec, request: Record<str
   const requestB64 = Buffer.from(JSON.stringify(request), "utf8").toString("base64");
   const evalScript = String.raw`
 const startedAt = Date.now();
+let input = {};
+let method = "GET";
+let path = "/";
+let baseUrl = "http://127.0.0.1:8080";
+let timeoutMs = 15000;
 try {
-  const input = JSON.parse(Buffer.from(process.env.AGENTRUN_LANE_REST_REQUEST_B64 || "", "base64").toString("utf8"));
+  input = JSON.parse(Buffer.from(process.env.AGENTRUN_LANE_REST_REQUEST_B64 || "", "base64").toString("utf8"));
+  method = String(input.method || "GET");
+  path = String(input.path || "/");
+  baseUrl = String(input.baseUrl || "http://127.0.0.1:8080");
+  timeoutMs = Number(input.timeoutMs || 15000);
   const headers = {};
   const authEnv = String(input.authEnv || "HWLAB_API_KEY");
   const apiKey = process.env[authEnv] || process.env.AGENTRUN_API_KEY || process.env.HWLAB_API_KEY || "";
@@ -6101,12 +6210,11 @@ try {
     body = JSON.stringify(input.body);
   }
   const controller = new AbortController();
-  const timeoutMs = Number(input.timeoutMs || 15000);
   const timeout = setTimeout(() => controller.abort(), timeoutMs);
   let response;
   try {
-    response = await fetch(new URL(String(input.path || "/"), String(input.baseUrl || "http://127.0.0.1:8080")).toString(), {
-      method: String(input.method || "GET"),
+    response = await fetch(new URL(path, baseUrl).toString(), {
+      method,
       headers,
       body,
       signal: controller.signal,
@@ -6117,7 +6225,10 @@ try {
   const text = await response.text();
   console.log(JSON.stringify({ ok: true, httpStatus: response.status, statusText: response.statusText, text, elapsedMs: Date.now() - startedAt, valuesPrinted: false }));
 } catch (error) {
-  console.log(JSON.stringify({ ok: false, failureKind: "manager-pod-fetch-failed", message: error instanceof Error ? error.message : String(error), elapsedMs: Date.now() - startedAt, valuesPrinted: false }));
+  const name = error instanceof Error ? error.name : "";
+  const message = error instanceof Error ? error.message : String(error);
+  const timedOut = name === "AbortError" || /abort|timed out|timeout/iu.test(message);
+  console.log(JSON.stringify({ ok: false, failureKind: timedOut ? "manager-pod-fetch-timeout" : "manager-pod-fetch-failed", message: timedOut ? "AgentRun manager request timed out for " + method + " " + path + " after " + timeoutMs + "ms" : message, elapsedMs: Date.now() - startedAt, path, baseUrl, valuesPrinted: false }));
 }
 `;
   return [
@@ -6733,7 +6844,7 @@ function safeAgentRunEnvelope(envelope: Record<string, unknown>): Record<string,
 }
 
 function normalizeAgentRunFailureKind(raw: string | null, httpStatus: number): AgentRunFailureKind {
-  if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed" || raw === "not-found") return raw;
+  if (raw === "auth-missing" || raw === "auth-failed" || raw === "agentrun-unreachable" || raw === "agentrun-timeout" || raw === "schema-mismatch" || raw === "unsupported-version" || raw === "validation-failed" || raw === "not-found") return raw;
   if (httpStatus === 401 || httpStatus === 403) return "auth-failed";
   if (httpStatus === 404) return "not-found";
   return raw === "schema-invalid" ? "validation-failed" : "validation-failed";
@@ -6823,7 +6934,7 @@ interface AgentRunResolvedAuth {
 }
 
 type AgentRunHttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
-type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "schema-mismatch" | "unsupported-version" | "validation-failed" | "not-found";
+type AgentRunFailureKind = "auth-missing" | "auth-failed" | "agentrun-unreachable" | "agentrun-timeout" | "schema-mismatch" | "unsupported-version" | "validation-failed" | "not-found";
 
 type AgentRunRestCompatGroup = "queue" | "sessions" | "aipod-specs" | "aipods" | "runs" | "commands" | "runner";
 type AgentRunResourceVerb = "get" | "describe" | "events" | "logs" | "result" | "ack" | "cancel" | "dispatch" | "create" | "apply" | "send" | "explain";

scripts/src/platform-infra-observability.ts

@@ -822,6 +822,10 @@ function renderTraceTable(input: {
     const row = asPlainRecord(item) ?? {};
     return [shortenEnd(textValue(row.name), 64), textValue(row.count)];
   });
+  const identity = asPlainRecord(input.result.identity) ?? {};
+  const identityRows = ["runId", "commandId", "runnerJobId", "runnerId", "backendProfile", "sourceCommit", "jobName", "podName"]
+    .map((key) => [key, joinValues(identity[key], 88)])
+    .filter((row) => row[1] !== "-");
   const next = asPlainRecord(input.result.next);
   const lines = [
     `platform-infra observability trace (${input.ok ? "ok" : "not-ok"})`,
@@ -835,6 +839,9 @@ function renderTraceTable(input: {
       joinValues(input.result.businessTraceIds, 34),
     ]]),
     "",
+    "Identity:",
+    formatTable(["FIELD", "VALUE"], identityRows.length > 0 ? identityRows : [["-", "-"]]),
+    "",
     "Key spans:",
     formatTable(["NAME", "SERVICE", "DETAIL"], spanRows.length > 0 ? spanRows : [["-", "-", "-"]]),
     "",
@@ -2125,6 +2132,7 @@ if not isinstance(parsed, dict):
 spans = []
 services = set()
 business_trace_ids = set()
+identity_values = collections.defaultdict(set)
 name_counts = collections.Counter()
 error_spans = []
 matched_spans = []
@@ -2146,6 +2154,11 @@ for batch in batches_from_body(parsed):
             attrs = item.get("attributes", {})
             if isinstance(attrs.get("traceId"), str):
                 business_trace_ids.add(attrs.get("traceId"))
+            if isinstance(attrs, dict):
+                for key in ("runId", "commandId", "runnerJobId", "runnerId", "backendProfile", "sourceCommit", "jobName", "podName"):
+                    value = attrs.get(key)
+                    if value not in (None, ""):
+                        identity_values[key].add(str(value))
             name = str(item.get("name") or "")
             name_counts[name] += 1
             spans.append(item)
@@ -2172,6 +2185,7 @@ payload = {
   "serviceCount": len(services),
   "services": sorted(services),
   "businessTraceIds": sorted(business_trace_ids)[:20],
+  "identity": {key: sorted(values)[:8] for key, values in identity_values.items() if values},
   "errorSpanCount": len(error_spans),
   "matchedSpanCount": len(matched_spans) if GREP is not None else None,
   "grep": GREP,
@@ -3009,6 +3023,9 @@ def http_status_summary(spans):
 def agentrun_summary(spans):
     terminal_spans = []
     latest_seq = None
+    failure_kind = None
+    failure_message = None
+    terminal_category = None
     seq_keys = ("maxSeq", "traceLastSeq", "endSeq", "seq", "eventSeq", "sourceSeq", "sourceLatestSeq", "latestSeq", "lastSeq")
     for item in spans:
         service = str(item.get("service") or "")
@@ -3024,9 +3041,17 @@ def agentrun_summary(spans):
             derived = terminal_status
             if not derived and name.startswith("runner_terminal."):
                 derived = name.split(".", 1)[1]
+            if attrs.get("failureKind") not in (None, ""):
+                failure_kind = attrs.get("failureKind")
+            if attrs.get("failureMessage") not in (None, ""):
+                failure_message = attrs.get("failureMessage")
+            if attrs.get("terminalCategory") not in (None, ""):
+                terminal_category = attrs.get("terminalCategory")
             terminal_spans.append({
                 "status": derived,
                 "eventType": attrs.get("eventType"),
+                "failureKind": attrs.get("failureKind"),
+                "terminalCategory": attrs.get("terminalCategory"),
                 "span": public_span(item),
             })
     terminal_status = terminal_spans[-1]["status"] if terminal_spans else None
@@ -3047,6 +3072,9 @@ def agentrun_summary(spans):
         runner_classification = "unknown"
     return {
         "terminalStatus": terminal_status,
+        "failureKind": failure_kind,
+        "failureMessage": failure_message,
+        "terminalCategory": terminal_category,
         "latestSeq": latest_seq,
         "terminalEventType": terminal_event_type,
         "terminalSpans": terminal_spans,
@@ -3176,6 +3204,19 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error
             "summary": "HWLAB projection stale is suspected because AgentRun is terminal but HWLAB read-model evidence is incomplete.",
             "evidence": lag_summary.get("reasons", []),
         })
+    if agentrun.get("terminalStatus") in ("failed", "error", "timeout", "blocked", "cancelled"):
+        candidates.append({
+            "code": "agentrun_terminal_failed",
+            "label": "AgentRun terminal failed",
+            "confidence": 0.9,
+            "summary": "AgentRun reached a non-success terminal state; inspect result/events/runnerjob before retrying or continuing the session.",
+            "evidence": {
+                "terminalStatus": agentrun.get("terminalStatus"),
+                "failureKind": agentrun.get("failureKind"),
+                "terminalCategory": agentrun.get("terminalCategory"),
+                "runnerProviderClassification": agentrun.get("runnerProviderClassification"),
+            },
+        })
     if agentrun.get("terminalStatus") == "completed":
         candidates.append({
             "code": "agentrun_completed_not_provider_blocked",
@@ -3451,11 +3492,18 @@ service_path["complete"] = all(service in services for service in expected_servi
 facts = []
 if http_summary.get("actorForbidden"):
     facts.append("actor forbidden")
-if agentrun.get("terminalStatus") == "completed":
+terminal_status = agentrun.get("terminalStatus")
+if terminal_status in ("failed", "error", "timeout", "blocked", "cancelled"):
+    failure_kind = agentrun.get("failureKind")
+    if failure_kind:
+        facts.append(f"AgentRun terminal failed ({failure_kind})")
+    else:
+        facts.append("AgentRun terminal failed")
+if terminal_status == "completed":
     facts.append("AgentRun completed")
 if lag.get("status") in ("confirmed", "suspected"):
     facts.append("HWLAB projection stale")
-if idle_warning_spans and agentrun.get("terminalStatus") in (None, ""):
+if idle_warning_spans and terminal_status in (None, ""):
     facts.append("AgentRun runner idle warnings active")
 if not facts:
     facts.append("no dominant Code Agent root cause classified")
@@ -3466,6 +3514,8 @@ summary = {
         "projectionLag": lag.get("status"),
         "runnerProvider": agentrun.get("runnerProviderClassification"),
         "actorForbidden": http_summary.get("actorForbidden"),
+        "terminalStatus": terminal_status,
+        "failureKind": agentrun.get("failureKind"),
     },
 }
 evidence = {