From 6ca6695c873727c00ed75bed7c65fc38bd56b3a0 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 19 Jun 2026 15:42:13 +0000 Subject: [PATCH] fix: surface HWLAB control-plane TaskRun failures --- scripts/src/hwlab-node-impl.ts | 231 ++++++++++++++++++++++++++++++++- scripts/src/jobs.ts | 102 ++++++++++++++- 2 files changed, 325 insertions(+), 8 deletions(-) diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 35d8d6b8..38f32cbe 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -2586,6 +2586,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType, degradedReason: pipelineRunDiagnostics.degradedReason ?? null, taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null, podCount: pipelineRunDiagnostics.podCount ?? null, + failedTaskRunCount: pipelineRunDiagnostics.failedTaskRunCount ?? null, + failedTaskRuns: pipelineRunDiagnostics.failedTaskRuns ?? [], + failureSummary: pipelineRunDiagnostics.failureSummary ?? null, pendingTaskRuns: pipelineRunDiagnostics.pendingTaskRuns ?? [], unscheduledPods: pipelineRunDiagnostics.unscheduledPods ?? [], schedulingMessages: pipelineRunDiagnostics.schedulingMessages ?? [], + next: pipelineRunDiagnostics.next ?? null, }, }, argo: { @@ -3463,6 +3471,14 @@ function nodeRuntimeStatusNextAction(status: Record, scoped: Re if (reason === "pipelinerun-not-succeeded") { return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`; } + if (reason === "node-runtime-ci-step-publish-failed") { + return `bun scripts/cli.ts platform-infra sub2api status --target ${scoped.node}`; + } + if (reason === "node-runtime-ci-taskrun-failed") { + const next = record(record(status.pipelineRunDiagnostics).next); + const failedStepLogs = typeof next.failedStepLogs === "string" ? next.failedStepLogs : null; + return failedStepLogs ?? `${nodeRuntimeStatusCommand(scoped)} --full`; + } if (reason === "node-runtime-ci-pod-capacity-exhausted" || reason === "node-runtime-ci-pod-unschedulable") { return `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane} --min-age-minutes 60 --limit 20 --dry-run`; } @@ -3491,11 +3507,28 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRuns(parseJsonRecordFromText(taskRunsResult.stdout)) : []; const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPods(parseJsonRecordFromText(podsResult.stdout)) : []; const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False"); + const failedTaskRuns = taskRuns.filter((item) => item.status === "False"); + const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods); + const stepPublishFailures = failedTaskRunSummaries.filter((item) => item.container === "step-publish" || item.step === "publish" || item.step === "step-publish"); const unscheduledPods = pods.filter((item) => item.scheduled === false); const schedulingMessages = unscheduledPods .map((item) => typeof item.scheduledMessage === "string" ? item.scheduledMessage : "") .filter((message) => message.length > 0); const tooManyPods = schedulingMessages.some((message) => /too many pods/iu.test(message)); + const failureSummary = failedTaskRunSummaries.length > 0 + ? { + failedTaskRunCount: failedTaskRuns.length, + failedStepCount: failedTaskRunSummaries.length, + stepPublishFailureCount: stepPublishFailures.length, + firstFailure: failedTaskRunSummaries[0] ?? null, + stepFailures: failedTaskRunSummaries.slice(0, 8), + nextAction: stepPublishFailures.length > 0 + ? "step-publish failed in a service build; first distinguish platform-infra Sub2API/proxy health from a single upstream transient, then choose controlled rerun or artifact-publish/envRecipe retry fix." + : failedTaskRunSummaries.length > 0 + ? "Inspect the failed TaskRun and bounded pod log command before rerunning the control-plane trigger." + : null, + } + : null; return { ok: taskRunsResult.exitCode === 0 && podsResult.exitCode === 0, pipelineRun, @@ -3503,6 +3536,10 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR pods, taskRunCount: taskRuns.length, podCount: pods.length, + failedTaskRunCount: failedTaskRuns.length, + failedTaskRuns: failedTaskRunSummaries, + stepPublishFailures, + failureSummary, pendingTaskRuns, unscheduledPods, schedulingMessages, @@ -3510,15 +3547,32 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR ? "node-runtime-ci-pod-capacity-exhausted" : unscheduledPods.length > 0 ? "node-runtime-ci-pod-unschedulable" - : pendingTaskRuns.length > 0 - ? "node-runtime-ci-taskrun-pending" - : undefined, + : stepPublishFailures.length > 0 + ? "node-runtime-ci-step-publish-failed" + : failedTaskRunSummaries.length > 0 + ? "node-runtime-ci-taskrun-failed" + : pendingTaskRuns.length > 0 + ? "node-runtime-ci-taskrun-pending" + : undefined, query: { taskRuns: compactRuntimeCommand(taskRunsResult), pods: compactRuntimeCommand(podsResult), }, next: tooManyPods || unscheduledPods.length > 0 ? { cleanupRuns: `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 60 --limit 20 --dry-run` } + : stepPublishFailures.length > 0 + ? { + sub2apiStatus: `bun scripts/cli.ts platform-infra sub2api status --target ${spec.nodeId}`, + sub2apiValidate: `bun scripts/cli.ts platform-infra sub2api validate --target ${spec.nodeId}`, + failedStepLogs: stepPublishFailures[0]?.logCommand ?? null, + rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm --rerun`, + } + : failedTaskRunSummaries.length > 0 + ? { + failedStepLogs: failedTaskRunSummaries[0]?.logCommand ?? null, + failedTaskRun: failedTaskRunSummaries[0]?.taskRunCommand ?? null, + status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`, + } : undefined, }; } @@ -3527,15 +3581,24 @@ function nodeRuntimePipelineDiagnosticTaskRuns(json: Record): A const items = Array.isArray(json.items) ? json.items.map(record) : []; return items.map((item) => { const metadata = record(item.metadata); + const labels = record(metadata.labels); + const spec = record(item.spec); + const taskRef = record(spec.taskRef); const status = record(item.status); const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : []; const condition = conditions[0] ?? {}; + const steps = nodeRuntimePipelineDiagnosticSteps(status.steps); + const failedSteps = steps.filter((step) => step.failed === true); return { name: metadata.name ?? null, + pipelineTask: labels["tekton.dev/pipelineTask"] ?? null, + taskRef: taskRef.name ?? null, status: condition.status ?? null, reason: condition.reason ?? null, - message: condition.message ?? null, + message: diagnosticText(condition.message), podName: status.podName ?? null, + steps, + failedSteps, }; }); } @@ -3544,21 +3607,169 @@ function nodeRuntimePipelineDiagnosticPods(json: Record): Array const items = Array.isArray(json.items) ? json.items.map(record) : []; return items.map((item) => { const metadata = record(item.metadata); + const labels = record(metadata.labels); const spec = record(item.spec); const status = record(item.status); const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : []; const scheduled = conditions.find((condition) => condition.type === "PodScheduled"); + const initContainers = nodeRuntimeContainerStatusSummaries(status.initContainerStatuses); + const containers = nodeRuntimeContainerStatusSummaries(status.containerStatuses); + const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true); return { name: metadata.name ?? null, + taskRun: labels["tekton.dev/taskRun"] ?? null, phase: status.phase ?? null, nodeName: spec.nodeName ?? null, scheduled: scheduled === undefined ? null : scheduled.status === "True", scheduledReason: scheduled?.reason ?? null, - scheduledMessage: scheduled?.message ?? null, + scheduledMessage: diagnosticText(scheduled?.message), + initContainers, + containers, + failedContainers, }; }); } +function nodeRuntimePipelineDiagnosticSteps(value: unknown): Array> { + const steps = Array.isArray(value) ? value.map(record) : []; + return steps.map((step) => { + const terminated = record(step.terminated); + const running = record(step.running); + const waiting = record(step.waiting); + const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null; + const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null; + const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null; + const state = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null; + return { + name: step.name ?? null, + container: step.container ?? null, + state, + failed: exitCode !== null && exitCode !== 0, + exitCode, + reason: terminatedReason ?? waitingReason, + message: diagnosticText(terminated.message ?? waiting.message), + startedAt: terminated.startedAt ?? running.startedAt ?? null, + finishedAt: terminated.finishedAt ?? null, + }; + }); +} + +function nodeRuntimeContainerStatusSummaries(value: unknown): Array> { + const containers = Array.isArray(value) ? value.map(record) : []; + return containers.map((container) => { + const state = record(container.state); + const terminated = record(state.terminated); + const waiting = record(state.waiting); + const running = record(state.running); + const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null; + const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null; + const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null; + const stateName = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null; + return { + name: container.name ?? null, + ready: container.ready === true, + restartCount: typeof container.restartCount === "number" ? container.restartCount : null, + state: stateName, + failed: exitCode !== null && exitCode !== 0, + exitCode, + reason: terminatedReason ?? waitingReason, + message: diagnosticText(terminated.message ?? waiting.message), + startedAt: terminated.startedAt ?? running.startedAt ?? null, + finishedAt: terminated.finishedAt ?? null, + }; + }); +} + +function nodeRuntimePipelineFailedTaskRunSummaries( + spec: HwlabRuntimeLaneSpec, + failedTaskRuns: Array>, + pods: Array>, +): Array> { + const summaries: Array> = []; + for (const taskRun of failedTaskRuns) { + const taskRunName = stringOrNull(taskRun.name); + const podName = stringOrNull(taskRun.podName); + const pod = pods.find((item) => item.name === podName || (taskRunName !== null && item.taskRun === taskRunName)) ?? {}; + const failedSteps = Array.isArray(taskRun.failedSteps) ? taskRun.failedSteps.map(record) : []; + const failedContainers = Array.isArray(pod.failedContainers) ? pod.failedContainers.map(record) : []; + const failures = failedSteps.length > 0 + ? failedSteps + : failedContainers.map((container) => ({ + name: typeof container.name === "string" && container.name.startsWith("step-") ? container.name.slice("step-".length) : container.name ?? null, + container: container.name ?? null, + state: container.state ?? null, + exitCode: container.exitCode ?? null, + reason: container.reason ?? null, + message: container.message ?? null, + })); + const effectiveFailures = failures.length > 0 ? failures : [{ name: null, container: null, state: null, exitCode: null, reason: taskRun.reason ?? null, message: taskRun.message ?? null }]; + for (const failure of effectiveFailures) { + const stepName = stringOrNull(failure.name); + const containerName = stringOrNull(failure.container) ?? (stepName === null ? null : `step-${stepName}`); + summaries.push({ + taskRun: taskRunName, + pipelineTask: taskRun.pipelineTask ?? null, + taskRef: taskRun.taskRef ?? null, + taskRunStatus: taskRun.status ?? null, + taskRunReason: taskRun.reason ?? null, + taskRunMessage: diagnosticText(taskRun.message), + pod: podName, + podPhase: pod.phase ?? null, + nodeName: pod.nodeName ?? null, + step: stepName, + container: containerName, + containerState: failure.state ?? null, + terminationReason: failure.reason ?? null, + exitCode: typeof failure.exitCode === "number" ? failure.exitCode : null, + message: diagnosticText(failure.message), + logCommand: podName === null ? null : nodeRuntimePipelineLogsCommand(spec, podName, containerName), + taskRunCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["get", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName, "-o", "yaml"]), + taskRunDescribeCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName]), + podDescribeCommand: podName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "pod", "-n", HWLAB_CI_NAMESPACE, podName]), + }); + } + } + return summaries.slice(0, 16); +} + +function nodeRuntimePipelineFailureSummary(value: unknown): Record | null { + const recordValue = record(value); + const direct = record(recordValue.failureSummary); + if (Object.keys(direct).length > 0) return direct; + const diagnostics = record(recordValue.diagnostics); + const fromDiagnostics = record(diagnostics.failureSummary); + return Object.keys(fromDiagnostics).length > 0 ? fromDiagnostics : null; +} + +function nodeRuntimePipelineLogsCommand(spec: HwlabRuntimeLaneSpec, podName: string, containerName: string | null): string { + return nodeRuntimeK3sCommand(spec, [ + "logs", + "--namespace", HWLAB_CI_NAMESPACE, + "--pod", podName, + ...(containerName === null ? ["--all-containers"] : ["--container", containerName]), + "--tail", "200", + ]); +} + +function nodeRuntimeK3sCommand(spec: HwlabRuntimeLaneSpec, args: string[]): string { + return ["trans", spec.nodeKubeRoute, ...args].map(shellQuote).join(" "); +} + +function stringOrNull(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function diagnosticText(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + if (trimmed.length === 0) return null; + return trimmed + .replace(/postgres(?:ql)?:\/\/[^\s"'`]+/giu, "postgres://") + .replace(/\b([A-Za-z0-9_.-]*(?:TOKEN|PASSWORD|SECRET|API_KEY|DATABASE_URL)[A-Za-z0-9_.-]*)=([^\s"'`]+)/giu, "$1=") + .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gu, "Bearer ") + .slice(0, 600); +} + function nodeRuntimeRenderToken(): string { return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-"); } @@ -4970,11 +5181,15 @@ function waitForNodeRuntimePipelineRunTerminal(spec: HwlabRuntimeLaneSpec, pipel printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt }); if (status === "True" || status === "False") { const ok = status === "True"; + const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun); + const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics); printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt }); return { ok, status: ok ? "succeeded" : "failed", pipelineRun: last, + diagnostics, + failureSummary, polls, elapsedMs: Date.now() - startedAt, degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed", @@ -4983,10 +5198,14 @@ function waitForNodeRuntimePipelineRunTerminal(spec: HwlabRuntimeLaneSpec, pipel sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now()))); } printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "timeout", pipelineRun, polls, elapsedMs: Date.now() - startedAt }); + const diagnostics = nodeRuntimePipelineRunDiagnostics(spec, pipelineRun); + const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics); return { ok: false, status: "timeout", pipelineRun: last, + diagnostics, + failureSummary, polls, elapsedMs: Date.now() - startedAt, degradedReason: "node-runtime-pipelinerun-wait-timeout", diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index a5c7cf79..d5102c6f 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -460,7 +460,11 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri const nextStatusCommand = hwlabRuntimeLaneStatusNextCommand(pipelineRun, lastEvent); if (nextStatusCommand.warning !== null) warnings.push(nextStatusCommand.warning); const tcpPoolDiagnostics = job.status === "succeeded" ? null : sshTcpPoolDiagnosticsFromJobText(`${stderrTail}\n${stdoutTail}`); + const pipelineFailureDiagnostics = runtimeLanePipelineFailureDiagnostics(stdoutTail); const slow = warnings.length > 0; + const diagnostics = pipelineFailureDiagnostics !== null + ? { pipelineFailure: pipelineFailureDiagnostics, sshTcpPool: tcpPoolDiagnostics } + : tcpPoolDiagnostics; return { kind: "hwlab-runtime-lane-trigger", stage, @@ -475,7 +479,7 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri eventsObserved: events.length, slow, warnings, - diagnostics: tcpPoolDiagnostics, + diagnostics, timings, summary: [ job.status, @@ -483,6 +487,7 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null, pipelineRun ? `pipelineRun=${pipelineRun}` : null, pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null, + pipelineFailureDiagnostics !== null ? runtimeLanePipelineFailureSummaryText(pipelineFailureDiagnostics) : null, elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null, lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null, @@ -674,6 +679,99 @@ function lastJsonLinePayload(text: string, prefix: string): Record | null { + const parsed = parseJsonObjectFromText(stdoutTail); + const data = recordField(parsed.data); + const directSummary = recordField(data.pipelineFailureSummary); + const waitSummary = recordField(recordField(data.pipelineWait).failureSummary); + const statusSummary = recordField(recordField(recordField(data.pipelineRun).diagnostics).failureSummary); + const failureSummary = Object.keys(directSummary).length > 0 + ? directSummary + : Object.keys(waitSummary).length > 0 + ? waitSummary + : statusSummary; + const firstFailure = recordField(failureSummary.firstFailure); + if (Object.keys(failureSummary).length > 0) { + return { + degradedReason: data.degradedReason ?? null, + failedTaskRunCount: failureSummary.failedTaskRunCount ?? null, + failedStepCount: failureSummary.failedStepCount ?? null, + stepPublishFailureCount: failureSummary.stepPublishFailureCount ?? null, + firstFailure: compactRuntimeLaneFailure(firstFailure), + nextAction: failureSummary.nextAction ?? null, + }; + } + const taskRun = firstMatch(stdoutTail, /"taskRun"\s*:\s*"([^"]+)"/u); + const pod = firstMatch(stdoutTail, /"pod"\s*:\s*"([^"]+)"/u); + const step = firstMatch(stdoutTail, /"step"\s*:\s*"([^"]+)"/u); + const container = firstMatch(stdoutTail, /"container"\s*:\s*"([^"]+)"/u); + if (taskRun === null && pod === null && step === null && container === null) return null; + return { + degradedReason: firstMatch(stdoutTail, /"degradedReason"\s*:\s*"([^"]+)"/u), + failedTaskRunCount: null, + failedStepCount: null, + stepPublishFailureCount: container === "step-publish" || step === "publish" || step === "step-publish" ? 1 : 0, + firstFailure: { taskRun, pod, step, container, logCommand: firstMatch(stdoutTail, /"logCommand"\s*:\s*"([^"]+)"/u) }, + nextAction: null, + }; +} + +function runtimeLanePipelineFailureSummaryText(diagnostics: Record): string | null { + const firstFailure = recordField(diagnostics.firstFailure); + const taskRun = stringField(firstFailure.taskRun); + const step = stringField(firstFailure.step); + const container = stringField(firstFailure.container); + const stepPublishCount = numberField(diagnostics.stepPublishFailureCount); + const failedTaskRunCount = numberField(diagnostics.failedTaskRunCount); + const parts = [ + failedTaskRunCount !== null ? `failedTaskRuns=${failedTaskRunCount}` : null, + taskRun !== null ? `taskRun=${taskRun}` : null, + step !== null ? `step=${step}` : null, + container !== null ? `container=${container}` : null, + stepPublishCount !== null && stepPublishCount > 0 ? "step-publish" : null, + ]; + const text = parts.filter(Boolean).join(" "); + return text.length > 0 ? text : null; +} + +function compactRuntimeLaneFailure(value: Record): Record | null { + if (Object.keys(value).length === 0) return null; + return { + taskRun: value.taskRun ?? null, + pipelineTask: value.pipelineTask ?? null, + taskRef: value.taskRef ?? null, + pod: value.pod ?? null, + step: value.step ?? null, + container: value.container ?? null, + terminationReason: value.terminationReason ?? null, + exitCode: value.exitCode ?? null, + logCommand: value.logCommand ?? null, + taskRunCommand: value.taskRunCommand ?? null, + podDescribeCommand: value.podDescribeCommand ?? null, + }; +} + +function parseJsonObjectFromText(text: string): Record { + const trimmed = text.trim(); + if (trimmed.length === 0) return {}; + try { + return recordField(JSON.parse(trimmed) as unknown); + } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start >= 0 && end > start) { + try { + return recordField(JSON.parse(trimmed.slice(start, end + 1)) as unknown); + } catch {} + } + } + return {}; +} + +function recordField(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary { const nowMs = Date.now(); const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000); @@ -801,7 +899,7 @@ function compactJobStdoutTail(job: JobRecord, progress: JobProgressSummary, rawT const interestingLines = rawTail.split(/\r?\n/u) .map((line) => line.trimEnd()) .filter((line) => !/"(stdoutTail|stderrTail|expected|result)"\s*:/u.test(line)) - .filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line)) + .filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|pipelineFailureSummary|failureSummary|firstFailure|failedTaskRun|stepFailure|taskRun|pod|step|container|terminationReason|exitCode|logCommand|taskRunCommand|podDescribeCommand|step-publish|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line)) .slice(-80); const header = [ `[stdout compacted: ${progress.kind}; requestedTailBytes=${requestedTailBytes}]`,