diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index 152bc5d0..4e5e435e 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -225,7 +225,7 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & { function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary { const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current"; const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush"; - if (!knownWorkflow && !gitMirrorWorkflow && tails === undefined) return genericJobProgress(job); + if (!knownWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail); const nowMs = Date.now(); const progressTailBytes = Math.max(4096, Math.floor(maxBytes)); const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes); @@ -349,10 +349,15 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder }; } -function genericJobProgress(job: JobRecord): JobProgressSummary { +function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary { const nowMs = Date.now(); - const stderrTail = tailFile(job.stderrFile, 96_000); - const downloadEvents = parseJsonLineEvents(stderrTail, "unidesk.ssh.download.progress"); + const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000); + const downloadProgressEvents = parseJsonLineEvents(stderrTail, "unidesk.ssh.download.progress"); + const downloadRetryEvents = [ + ...parseJsonLineEvents(stderrTail, "unidesk.ssh.download.empty-read-retry"), + ...parseJsonLineEvents(stderrTail, "unidesk.ssh.download.short-read-retry"), + ]; + const downloadEvents = [...downloadProgressEvents, ...downloadRetryEvents].sort((left, right) => Number(left.blockIndex ?? -1) - Number(right.blockIndex ?? -1)); const lastDownload = downloadEvents.at(-1) ?? null; const lastDownloadAt = stringField(lastDownload?.at); const lastEventAgeSeconds = lastDownloadAt === null ? null : secondsSince(lastDownloadAt, job.finishedAt ?? nowMs); @@ -368,7 +373,7 @@ function genericJobProgress(job: JobRecord): JobProgressSummary { }); const downloadSummary = lastDownload === null ? null - : ` ssh-download ${Number(lastDownload.actualBytes ?? 0)}/${Number(lastDownload.expectedBytes ?? 0)} bytes chunks=${Number(lastDownload.chunks ?? 0)}`; + : ` ssh-download ${Number(lastDownload.actualBytes ?? 0)}/${Number(lastDownload.expectedBytes ?? 0)} bytes chunks=${Number(lastDownload.chunks ?? 0)}${downloadRetryEvents.length === 0 ? "" : ` retries=${downloadRetryEvents.length} lastRetry=${String(lastDownload.reason ?? "unknown")}`}`; return { kind: "generic", stage: lastDownload === null ? null : "ssh-download",