fix: surface HWLAB control-plane TaskRun failures

2026-06-19 15:42:13 +00:00
parent 20c39269ce
commit 6ca6695c87
2 changed files with 325 additions and 8 deletions
@@ -460,7 +460,11 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri
  const nextStatusCommand = hwlabRuntimeLaneStatusNextCommand(pipelineRun, lastEvent);
  if (nextStatusCommand.warning !== null) warnings.push(nextStatusCommand.warning);
  const tcpPoolDiagnostics = job.status === "succeeded" ? null : sshTcpPoolDiagnosticsFromJobText(`${stderrTail}\n${stdoutTail}`);
+  const pipelineFailureDiagnostics = runtimeLanePipelineFailureDiagnostics(stdoutTail);
  const slow = warnings.length > 0;
+  const diagnostics = pipelineFailureDiagnostics !== null
+    ? { pipelineFailure: pipelineFailureDiagnostics, sshTcpPool: tcpPoolDiagnostics }
+    : tcpPoolDiagnostics;
  return {
    kind: "hwlab-runtime-lane-trigger",
    stage,
@@ -475,7 +479,7 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri
    eventsObserved: events.length,
    slow,
    warnings,
-    diagnostics: tcpPoolDiagnostics,
+    diagnostics,
    timings,
    summary: [
      job.status,
@@ -483,6 +487,7 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri
      sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
      pipelineRun ? `pipelineRun=${pipelineRun}` : null,
      pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null,
+      pipelineFailureDiagnostics !== null ? runtimeLanePipelineFailureSummaryText(pipelineFailureDiagnostics) : null,
      elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
      stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
      lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
@@ -674,6 +679,99 @@ function lastJsonLinePayload(text: string, prefix: string): Record<string, unkno
  return null;
 }

+function runtimeLanePipelineFailureDiagnostics(stdoutTail: string): Record<string, unknown> | null {
+  const parsed = parseJsonObjectFromText(stdoutTail);
+  const data = recordField(parsed.data);
+  const directSummary = recordField(data.pipelineFailureSummary);
+  const waitSummary = recordField(recordField(data.pipelineWait).failureSummary);
+  const statusSummary = recordField(recordField(recordField(data.pipelineRun).diagnostics).failureSummary);
+  const failureSummary = Object.keys(directSummary).length > 0
+    ? directSummary
+    : Object.keys(waitSummary).length > 0
+      ? waitSummary
+      : statusSummary;
+  const firstFailure = recordField(failureSummary.firstFailure);
+  if (Object.keys(failureSummary).length > 0) {
+    return {
+      degradedReason: data.degradedReason ?? null,
+      failedTaskRunCount: failureSummary.failedTaskRunCount ?? null,
+      failedStepCount: failureSummary.failedStepCount ?? null,
+      stepPublishFailureCount: failureSummary.stepPublishFailureCount ?? null,
+      firstFailure: compactRuntimeLaneFailure(firstFailure),
+      nextAction: failureSummary.nextAction ?? null,
+    };
+  }
+  const taskRun = firstMatch(stdoutTail, /"taskRun"\s*:\s*"([^"]+)"/u);
+  const pod = firstMatch(stdoutTail, /"pod"\s*:\s*"([^"]+)"/u);
+  const step = firstMatch(stdoutTail, /"step"\s*:\s*"([^"]+)"/u);
+  const container = firstMatch(stdoutTail, /"container"\s*:\s*"([^"]+)"/u);
+  if (taskRun === null && pod === null && step === null && container === null) return null;
+  return {
+    degradedReason: firstMatch(stdoutTail, /"degradedReason"\s*:\s*"([^"]+)"/u),
+    failedTaskRunCount: null,
+    failedStepCount: null,
+    stepPublishFailureCount: container === "step-publish" || step === "publish" || step === "step-publish" ? 1 : 0,
+    firstFailure: { taskRun, pod, step, container, logCommand: firstMatch(stdoutTail, /"logCommand"\s*:\s*"([^"]+)"/u) },
+    nextAction: null,
+  };
+}
+
+function runtimeLanePipelineFailureSummaryText(diagnostics: Record<string, unknown>): string | null {
+  const firstFailure = recordField(diagnostics.firstFailure);
+  const taskRun = stringField(firstFailure.taskRun);
+  const step = stringField(firstFailure.step);
+  const container = stringField(firstFailure.container);
+  const stepPublishCount = numberField(diagnostics.stepPublishFailureCount);
+  const failedTaskRunCount = numberField(diagnostics.failedTaskRunCount);
+  const parts = [
+    failedTaskRunCount !== null ? `failedTaskRuns=${failedTaskRunCount}` : null,
+    taskRun !== null ? `taskRun=${taskRun}` : null,
+    step !== null ? `step=${step}` : null,
+    container !== null ? `container=${container}` : null,
+    stepPublishCount !== null && stepPublishCount > 0 ? "step-publish" : null,
+  ];
+  const text = parts.filter(Boolean).join(" ");
+  return text.length > 0 ? text : null;
+}
+
+function compactRuntimeLaneFailure(value: Record<string, unknown>): Record<string, unknown> | null {
+  if (Object.keys(value).length === 0) return null;
+  return {
+    taskRun: value.taskRun ?? null,
+    pipelineTask: value.pipelineTask ?? null,
+    taskRef: value.taskRef ?? null,
+    pod: value.pod ?? null,
+    step: value.step ?? null,
+    container: value.container ?? null,
+    terminationReason: value.terminationReason ?? null,
+    exitCode: value.exitCode ?? null,
+    logCommand: value.logCommand ?? null,
+    taskRunCommand: value.taskRunCommand ?? null,
+    podDescribeCommand: value.podDescribeCommand ?? null,
+  };
+}
+
+function parseJsonObjectFromText(text: string): Record<string, unknown> {
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return {};
+  try {
+    return recordField(JSON.parse(trimmed) as unknown);
+  } catch {
+    const start = trimmed.indexOf("{");
+    const end = trimmed.lastIndexOf("}");
+    if (start >= 0 && end > start) {
+      try {
+        return recordField(JSON.parse(trimmed.slice(start, end + 1)) as unknown);
+      } catch {}
+    }
+  }
+  return {};
+}
+
+function recordField(value: unknown): Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
+}
+
 function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary {
  const nowMs = Date.now();
  const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000);
@@ -801,7 +899,7 @@ function compactJobStdoutTail(job: JobRecord, progress: JobProgressSummary, rawT
  const interestingLines = rawTail.split(/\r?\n/u)
    .map((line) => line.trimEnd())
    .filter((line) => !/"(stdoutTail|stderrTail|expected|result)"\s*:/u.test(line))
-    .filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line))
+    .filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|pipelineFailureSummary|failureSummary|firstFailure|failedTaskRun|stepFailure|taskRun|pod|step|container|terminationReason|exitCode|logCommand|taskRunCommand|podDescribeCommand|step-publish|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line))
    .slice(-80);
  const header = [
    `[stdout compacted: ${progress.kind}; requestedTailBytes=${requestedTailBytes}]`,