Merge pull request #499 from pikasTech/fix/hwlab-control-plane-status-484

修复 HWLAB control-plane 失败 TaskRun 摘要

scripts/src/hwlab-node-impl.ts

@@ -2586,6 +2586,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
       ? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds)
       : { ok: true, status: "already-succeeded", pipelineRun: before, polls: 0, elapsedMs: 0 };
     const waitedPipelineRun = record(pipelineWait.pipelineRun);
+    const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
     const postFlush = waitedPipelineRun.status === "True"
       ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
       : null;
@@ -2600,6 +2601,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
       pipelineRun,
       before,
       pipelineWait,
+      pipelineFailureSummary,
       postFlush,
       skipped: true,
       reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
@@ -2665,6 +2667,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
   printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined });
   const pipelineWait = createOk ? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds) : null;
   const waitedPipelineRun = record(pipelineWait?.pipelineRun);
+  const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
   const postFlush = waitedPipelineRun.status === "True"
     ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
     : null;
@@ -2687,6 +2690,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
     create: compactRuntimeCommand(create),
     after,
     pipelineWait,
+    pipelineFailureSummary,
     postFlush,
     createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined,
     degradedReason: ok
@@ -3211,7 +3215,7 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
   const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60);
   const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
   const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
-  const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.status === "Unknown"
+  const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
     ? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
     : null;
   const workloads = namespaceExists
@@ -3401,9 +3405,13 @@ function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>,
         degradedReason: pipelineRunDiagnostics.degradedReason ?? null,
         taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null,
         podCount: pipelineRunDiagnostics.podCount ?? null,
+        failedTaskRunCount: pipelineRunDiagnostics.failedTaskRunCount ?? null,
+        failedTaskRuns: pipelineRunDiagnostics.failedTaskRuns ?? [],
+        failureSummary: pipelineRunDiagnostics.failureSummary ?? null,
         pendingTaskRuns: pipelineRunDiagnostics.pendingTaskRuns ?? [],
         unscheduledPods: pipelineRunDiagnostics.unscheduledPods ?? [],
         schedulingMessages: pipelineRunDiagnostics.schedulingMessages ?? [],
+        next: pipelineRunDiagnostics.next ?? null,
       },
     },
     argo: {
@@ -3463,6 +3471,14 @@ function nodeRuntimeStatusNextAction(status: Record<string, unknown>, scoped: Re
   if (reason === "pipelinerun-not-succeeded") {
     return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`;
   }
+  if (reason === "node-runtime-ci-step-publish-failed") {
+    return `bun scripts/cli.ts platform-infra sub2api status --target ${scoped.node}`;
+  }
+  if (reason === "node-runtime-ci-taskrun-failed") {
+    const next = record(record(status.pipelineRunDiagnostics).next);
+    const failedStepLogs = typeof next.failedStepLogs === "string" ? next.failedStepLogs : null;
+    return failedStepLogs ?? `${nodeRuntimeStatusCommand(scoped)} --full`;
+  }
   if (reason === "node-runtime-ci-pod-capacity-exhausted" || reason === "node-runtime-ci-pod-unschedulable") {
     return `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane} --min-age-minutes 60 --limit 20 --dry-run`;
   }
@@ -3491,11 +3507,28 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR
   const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRuns(parseJsonRecordFromText(taskRunsResult.stdout)) : [];
   const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPods(parseJsonRecordFromText(podsResult.stdout)) : [];
   const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False");
+  const failedTaskRuns = taskRuns.filter((item) => item.status === "False");
+  const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods);
+  const stepPublishFailures = failedTaskRunSummaries.filter((item) => item.container === "step-publish" || item.step === "publish" || item.step === "step-publish");
   const unscheduledPods = pods.filter((item) => item.scheduled === false);
   const schedulingMessages = unscheduledPods
     .map((item) => typeof item.scheduledMessage === "string" ? item.scheduledMessage : "")
     .filter((message) => message.length > 0);
   const tooManyPods = schedulingMessages.some((message) => /too many pods/iu.test(message));
+  const failureSummary = failedTaskRunSummaries.length > 0
+    ? {
+      failedTaskRunCount: failedTaskRuns.length,
+      failedStepCount: failedTaskRunSummaries.length,
+      stepPublishFailureCount: stepPublishFailures.length,
+      firstFailure: failedTaskRunSummaries[0] ?? null,
+      stepFailures: failedTaskRunSummaries.slice(0, 8),
+      nextAction: stepPublishFailures.length > 0
+        ? "step-publish failed in a service build; first distinguish platform-infra Sub2API/proxy health from a single upstream transient, then choose controlled rerun or artifact-publish/envRecipe retry fix."
+        : failedTaskRunSummaries.length > 0
+          ? "Inspect the failed TaskRun and bounded pod log command before rerunning the control-plane trigger."
+          : null,
+    }
+    : null;
   return {
     ok: taskRunsResult.exitCode === 0 && podsResult.exitCode === 0,
     pipelineRun,
@@ -3503,6 +3536,10 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR
     pods,
     taskRunCount: taskRuns.length,
     podCount: pods.length,
+    failedTaskRunCount: failedTaskRuns.length,
+    failedTaskRuns: failedTaskRunSummaries,
+    stepPublishFailures,
+    failureSummary,
     pendingTaskRuns,
     unscheduledPods,
     schedulingMessages,
@@ -3510,15 +3547,32 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR
       ? "node-runtime-ci-pod-capacity-exhausted"
       : unscheduledPods.length > 0
         ? "node-runtime-ci-pod-unschedulable"
-        : pendingTaskRuns.length > 0
-          ? "node-runtime-ci-taskrun-pending"
-          : undefined,
+        : stepPublishFailures.length > 0
+          ? "node-runtime-ci-step-publish-failed"
+          : failedTaskRunSummaries.length > 0
+            ? "node-runtime-ci-taskrun-failed"
+            : pendingTaskRuns.length > 0
+              ? "node-runtime-ci-taskrun-pending"
+              : undefined,
     query: {
       taskRuns: compactRuntimeCommand(taskRunsResult),
       pods: compactRuntimeCommand(podsResult),
     },
     next: tooManyPods || unscheduledPods.length > 0
       ? { cleanupRuns: `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 60 --limit 20 --dry-run` }
+      : stepPublishFailures.length > 0
+        ? {
+          sub2apiStatus: `bun scripts/cli.ts platform-infra sub2api status --target ${spec.nodeId}`,
+          sub2apiValidate: `bun scripts/cli.ts platform-infra sub2api validate --target ${spec.nodeId}`,
+          failedStepLogs: stepPublishFailures[0]?.logCommand ?? null,
+          rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm --rerun`,
+        }
+        : failedTaskRunSummaries.length > 0
+          ? {
+            failedStepLogs: failedTaskRunSummaries[0]?.logCommand ?? null,
+            failedTaskRun: failedTaskRunSummaries[0]?.taskRunCommand ?? null,
+            status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
+          }
       : undefined,
   };
 }
@@ -3527,15 +3581,24 @@ function nodeRuntimePipelineDiagnosticTaskRuns(json: Record<string, unknown>): A
   const items = Array.isArray(json.items) ? json.items.map(record) : [];
   return items.map((item) => {
     const metadata = record(item.metadata);
+    const labels = record(metadata.labels);
+    const spec = record(item.spec);
+    const taskRef = record(spec.taskRef);
     const status = record(item.status);
     const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
     const condition = conditions[0] ?? {};
+    const steps = nodeRuntimePipelineDiagnosticSteps(status.steps);
+    const failedSteps = steps.filter((step) => step.failed === true);
     return {
       name: metadata.name ?? null,
+      pipelineTask: labels["tekton.dev/pipelineTask"] ?? null,
+      taskRef: taskRef.name ?? null,
       status: condition.status ?? null,
       reason: condition.reason ?? null,
-      message: condition.message ?? null,
+      message: diagnosticText(condition.message),
       podName: status.podName ?? null,
+      steps,
+      failedSteps,
     };
   });
 }
@@ -3544,21 +3607,169 @@ function nodeRuntimePipelineDiagnosticPods(json: Record<string, unknown>): Array
   const items = Array.isArray(json.items) ? json.items.map(record) : [];
   return items.map((item) => {
     const metadata = record(item.metadata);
+    const labels = record(metadata.labels);
     const spec = record(item.spec);
     const status = record(item.status);
     const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
     const scheduled = conditions.find((condition) => condition.type === "PodScheduled");
+    const initContainers = nodeRuntimeContainerStatusSummaries(status.initContainerStatuses);
+    const containers = nodeRuntimeContainerStatusSummaries(status.containerStatuses);
+    const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true);
     return {
       name: metadata.name ?? null,
+      taskRun: labels["tekton.dev/taskRun"] ?? null,
       phase: status.phase ?? null,
       nodeName: spec.nodeName ?? null,
       scheduled: scheduled === undefined ? null : scheduled.status === "True",
       scheduledReason: scheduled?.reason ?? null,
-      scheduledMessage: scheduled?.message ?? null,
+      scheduledMessage: diagnosticText(scheduled?.message),
+      initContainers,
+      containers,
+      failedContainers,
     };
   });
 }
 
+function nodeRuntimePipelineDiagnosticSteps(value: unknown): Array<Record<string, unknown>> {
+  const steps = Array.isArray(value) ? value.map(record) : [];
+  return steps.map((step) => {
+    const terminated = record(step.terminated);
+    const running = record(step.running);
+    const waiting = record(step.waiting);
+    const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
+    const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
+    const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
+    const state = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
+    return {
+      name: step.name ?? null,
+      container: step.container ?? null,
+      state,
+      failed: exitCode !== null && exitCode !== 0,
+      exitCode,
+      reason: terminatedReason ?? waitingReason,
+      message: diagnosticText(terminated.message ?? waiting.message),
+      startedAt: terminated.startedAt ?? running.startedAt ?? null,
+      finishedAt: terminated.finishedAt ?? null,
+    };
+  });
+}
+
+function nodeRuntimeContainerStatusSummaries(value: unknown): Array<Record<string, unknown>> {
+  const containers = Array.isArray(value) ? value.map(record) : [];
+  return containers.map((container) => {
+    const state = record(container.state);
+    const terminated = record(state.terminated);
+    const waiting = record(state.waiting);
+    const running = record(state.running);
+    const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
+    const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
+    const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
+    const stateName = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
+    return {
+      name: container.name ?? null,
+      ready: container.ready === true,
+      restartCount: typeof container.restartCount === "number" ? container.restartCount : null,
+      state: stateName,
+      failed: exitCode !== null && exitCode !== 0,
+      exitCode,
+      reason: terminatedReason ?? waitingReason,
+      message: diagnosticText(terminated.message ?? waiting.message),
+      startedAt: terminated.startedAt ?? running.startedAt ?? null,
+      finishedAt: terminated.finishedAt ?? null,
+    };
+  });
+}
+
+function nodeRuntimePipelineFailedTaskRunSummaries(
+  spec: HwlabRuntimeLaneSpec,
+  failedTaskRuns: Array<Record<string, unknown>>,
+  pods: Array<Record<string, unknown>>,
+): Array<Record<string, unknown>> {
+  const summaries: Array<Record<string, unknown>> = [];
+  for (const taskRun of failedTaskRuns) {
+    const taskRunName = stringOrNull(taskRun.name);
+    const podName = stringOrNull(taskRun.podName);
+    const pod = pods.find((item) => item.name === podName || (taskRunName !== null && item.taskRun === taskRunName)) ?? {};
+    const failedSteps = Array.isArray(taskRun.failedSteps) ? taskRun.failedSteps.map(record) : [];
+    const failedContainers = Array.isArray(pod.failedContainers) ? pod.failedContainers.map(record) : [];
+    const failures = failedSteps.length > 0
+      ? failedSteps
+      : failedContainers.map((container) => ({
+        name: typeof container.name === "string" && container.name.startsWith("step-") ? container.name.slice("step-".length) : container.name ?? null,
+        container: container.name ?? null,
+        state: container.state ?? null,
+        exitCode: container.exitCode ?? null,
+        reason: container.reason ?? null,
+        message: container.message ?? null,
+      }));
+    const effectiveFailures = failures.length > 0 ? failures : [{ name: null, container: null, state: null, exitCode: null, reason: taskRun.reason ?? null, message: taskRun.message ?? null }];
+    for (const failure of effectiveFailures) {
+      const stepName = stringOrNull(failure.name);
+      const containerName = stringOrNull(failure.container) ?? (stepName === null ? null : `step-${stepName}`);
+      summaries.push({
+        taskRun: taskRunName,
+        pipelineTask: taskRun.pipelineTask ?? null,
+        taskRef: taskRun.taskRef ?? null,
+        taskRunStatus: taskRun.status ?? null,
+        taskRunReason: taskRun.reason ?? null,
+        taskRunMessage: diagnosticText(taskRun.message),
+        pod: podName,
+        podPhase: pod.phase ?? null,
+        nodeName: pod.nodeName ?? null,
+        step: stepName,
+        container: containerName,
+        containerState: failure.state ?? null,
+        terminationReason: failure.reason ?? null,
+        exitCode: typeof failure.exitCode === "number" ? failure.exitCode : null,
+        message: diagnosticText(failure.message),
+        logCommand: podName === null ? null : nodeRuntimePipelineLogsCommand(spec, podName, containerName),
+        taskRunCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["get", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName, "-o", "yaml"]),
+        taskRunDescribeCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName]),
+        podDescribeCommand: podName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "pod", "-n", HWLAB_CI_NAMESPACE, podName]),
+      });
+    }
+  }
+  return summaries.slice(0, 16);
+}
+
+function nodeRuntimePipelineFailureSummary(value: unknown): Record<string, unknown> | null {
+  const recordValue = record(value);
+  const direct = record(recordValue.failureSummary);
+  if (Object.keys(direct).length > 0) return direct;
+  const diagnostics = record(recordValue.diagnostics);
+  const fromDiagnostics = record(diagnostics.failureSummary);
+  return Object.keys(fromDiagnostics).length > 0 ? fromDiagnostics : null;
+}
+
+function nodeRuntimePipelineLogsCommand(spec: HwlabRuntimeLaneSpec, podName: string, containerName: string | null): string {
+  return nodeRuntimeK3sCommand(spec, [
+    "logs",
+    "--namespace", HWLAB_CI_NAMESPACE,
+    "--pod", podName,
+    ...(containerName === null ? ["--all-containers"] : ["--container", containerName]),
+    "--tail", "200",
+  ]);
+}
+
+function nodeRuntimeK3sCommand(spec: HwlabRuntimeLaneSpec, args: string[]): string {
+  return ["trans", spec.nodeKubeRoute, ...args].map(shellQuote).join(" ");
+}
+
+function stringOrNull(value: unknown): string | null {
+  return typeof value === "string" && value.length > 0 ? value : null;
+}
+
+function diagnosticText(value: unknown): string | null {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  if (trimmed.length === 0) return null;
+  return trimmed
+    .replace(/postgres(?:ql)?:\/\/[^\s"'`]+/giu, "postgres://<redacted>")
+    .replace(/\b([A-Za-z0-9_.-]*(?:TOKEN|PASSWORD|SECRET|API_KEY|DATABASE_URL)[A-Za-z0-9_.-]*)=([^\s"'`]+)/giu, "$1=<redacted>")
+    .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gu, "Bearer <redacted>")
+    .slice(0, 600);
+}
+
 function nodeRuntimeRenderToken(): string {
   return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-");
 }
@@ -4970,11 +5181,15 @@ function waitForNodeRuntimePipelineRunTerminal(spec: HwlabRuntimeLaneSpec, pipel
     printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
     if (status === "True" || status === "False") {
       const ok = status === "True";
+      const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
+      const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
       printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
       return {
         ok,
         status: ok ? "succeeded" : "failed",
         pipelineRun: last,
+        diagnostics,
+        failureSummary,
         polls,
         elapsedMs: Date.now() - startedAt,
         degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
@@ -4983,10 +5198,14 @@ function waitForNodeRuntimePipelineRunTerminal(spec: HwlabRuntimeLaneSpec, pipel
     sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now())));
   }
   printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "timeout", pipelineRun, polls, elapsedMs: Date.now() - startedAt });
+  const diagnostics = nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
+  const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
   return {
     ok: false,
     status: "timeout",
     pipelineRun: last,
+    diagnostics,
+    failureSummary,
     polls,
     elapsedMs: Date.now() - startedAt,
     degradedReason: "node-runtime-pipelinerun-wait-timeout",

scripts/src/jobs.ts

@@ -460,7 +460,11 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri
   const nextStatusCommand = hwlabRuntimeLaneStatusNextCommand(pipelineRun, lastEvent);
   if (nextStatusCommand.warning !== null) warnings.push(nextStatusCommand.warning);
   const tcpPoolDiagnostics = job.status === "succeeded" ? null : sshTcpPoolDiagnosticsFromJobText(`${stderrTail}\n${stdoutTail}`);
+  const pipelineFailureDiagnostics = runtimeLanePipelineFailureDiagnostics(stdoutTail);
   const slow = warnings.length > 0;
+  const diagnostics = pipelineFailureDiagnostics !== null
+    ? { pipelineFailure: pipelineFailureDiagnostics, sshTcpPool: tcpPoolDiagnostics }
+    : tcpPoolDiagnostics;
   return {
     kind: "hwlab-runtime-lane-trigger",
     stage,
@@ -475,7 +479,7 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri
     eventsObserved: events.length,
     slow,
     warnings,
-    diagnostics: tcpPoolDiagnostics,
+    diagnostics,
     timings,
     summary: [
       job.status,
@@ -483,6 +487,7 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri
       sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
       pipelineRun ? `pipelineRun=${pipelineRun}` : null,
       pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null,
+      pipelineFailureDiagnostics !== null ? runtimeLanePipelineFailureSummaryText(pipelineFailureDiagnostics) : null,
       elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
       stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
       lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
@@ -674,6 +679,99 @@ function lastJsonLinePayload(text: string, prefix: string): Record<string, unkno
   return null;
 }
 
+function runtimeLanePipelineFailureDiagnostics(stdoutTail: string): Record<string, unknown> | null {
+  const parsed = parseJsonObjectFromText(stdoutTail);
+  const data = recordField(parsed.data);
+  const directSummary = recordField(data.pipelineFailureSummary);
+  const waitSummary = recordField(recordField(data.pipelineWait).failureSummary);
+  const statusSummary = recordField(recordField(recordField(data.pipelineRun).diagnostics).failureSummary);
+  const failureSummary = Object.keys(directSummary).length > 0
+    ? directSummary
+    : Object.keys(waitSummary).length > 0
+      ? waitSummary
+      : statusSummary;
+  const firstFailure = recordField(failureSummary.firstFailure);
+  if (Object.keys(failureSummary).length > 0) {
+    return {
+      degradedReason: data.degradedReason ?? null,
+      failedTaskRunCount: failureSummary.failedTaskRunCount ?? null,
+      failedStepCount: failureSummary.failedStepCount ?? null,
+      stepPublishFailureCount: failureSummary.stepPublishFailureCount ?? null,
+      firstFailure: compactRuntimeLaneFailure(firstFailure),
+      nextAction: failureSummary.nextAction ?? null,
+    };
+  }
+  const taskRun = firstMatch(stdoutTail, /"taskRun"\s*:\s*"([^"]+)"/u);
+  const pod = firstMatch(stdoutTail, /"pod"\s*:\s*"([^"]+)"/u);
+  const step = firstMatch(stdoutTail, /"step"\s*:\s*"([^"]+)"/u);
+  const container = firstMatch(stdoutTail, /"container"\s*:\s*"([^"]+)"/u);
+  if (taskRun === null && pod === null && step === null && container === null) return null;
+  return {
+    degradedReason: firstMatch(stdoutTail, /"degradedReason"\s*:\s*"([^"]+)"/u),
+    failedTaskRunCount: null,
+    failedStepCount: null,
+    stepPublishFailureCount: container === "step-publish" || step === "publish" || step === "step-publish" ? 1 : 0,
+    firstFailure: { taskRun, pod, step, container, logCommand: firstMatch(stdoutTail, /"logCommand"\s*:\s*"([^"]+)"/u) },
+    nextAction: null,
+  };
+}
+
+function runtimeLanePipelineFailureSummaryText(diagnostics: Record<string, unknown>): string | null {
+  const firstFailure = recordField(diagnostics.firstFailure);
+  const taskRun = stringField(firstFailure.taskRun);
+  const step = stringField(firstFailure.step);
+  const container = stringField(firstFailure.container);
+  const stepPublishCount = numberField(diagnostics.stepPublishFailureCount);
+  const failedTaskRunCount = numberField(diagnostics.failedTaskRunCount);
+  const parts = [
+    failedTaskRunCount !== null ? `failedTaskRuns=${failedTaskRunCount}` : null,
+    taskRun !== null ? `taskRun=${taskRun}` : null,
+    step !== null ? `step=${step}` : null,
+    container !== null ? `container=${container}` : null,
+    stepPublishCount !== null && stepPublishCount > 0 ? "step-publish" : null,
+  ];
+  const text = parts.filter(Boolean).join(" ");
+  return text.length > 0 ? text : null;
+}
+
+function compactRuntimeLaneFailure(value: Record<string, unknown>): Record<string, unknown> | null {
+  if (Object.keys(value).length === 0) return null;
+  return {
+    taskRun: value.taskRun ?? null,
+    pipelineTask: value.pipelineTask ?? null,
+    taskRef: value.taskRef ?? null,
+    pod: value.pod ?? null,
+    step: value.step ?? null,
+    container: value.container ?? null,
+    terminationReason: value.terminationReason ?? null,
+    exitCode: value.exitCode ?? null,
+    logCommand: value.logCommand ?? null,
+    taskRunCommand: value.taskRunCommand ?? null,
+    podDescribeCommand: value.podDescribeCommand ?? null,
+  };
+}
+
+function parseJsonObjectFromText(text: string): Record<string, unknown> {
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return {};
+  try {
+    return recordField(JSON.parse(trimmed) as unknown);
+  } catch {
+    const start = trimmed.indexOf("{");
+    const end = trimmed.lastIndexOf("}");
+    if (start >= 0 && end > start) {
+      try {
+        return recordField(JSON.parse(trimmed.slice(start, end + 1)) as unknown);
+      } catch {}
+    }
+  }
+  return {};
+}
+
+function recordField(value: unknown): Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
+}
+
 function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary {
   const nowMs = Date.now();
   const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000);
@@ -801,7 +899,7 @@ function compactJobStdoutTail(job: JobRecord, progress: JobProgressSummary, rawT
   const interestingLines = rawTail.split(/\r?\n/u)
     .map((line) => line.trimEnd())
     .filter((line) => !/"(stdoutTail|stderrTail|expected|result)"\s*:/u.test(line))
-    .filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line))
+    .filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|pipelineFailureSummary|failureSummary|firstFailure|failedTaskRun|stepFailure|taskRun|pod|step|container|terminationReason|exitCode|logCommand|taskRunCommand|podDescribeCommand|step-publish|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line))
     .slice(-80);
   const header = [
     `[stdout compacted: ${progress.kind}; requestedTailBytes=${requestedTailBytes}]`,