// SPEC: PJ2026-01060509 出站诊断 draft-2026-06-26-p8-egress-job-friction. // Async job state, summaries, and lifecycle drill-down rendering for UniDesk CLI. import { spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { repoRoot, rootPath } from "./config"; import { runCommandToFiles, tailFile } from "./command"; import { hwlabDefaultRuntimeTarget } from "./hwlab-node-lanes"; import type { RenderedCliResult } from "./output"; export type JobStatus = "queued" | "running" | "succeeded" | "failed" | "canceled"; export interface JobRecord { id: string; name: string; status: JobStatus; command: string[]; cwd: string; runner: "local" | "docker"; runnerPid?: number | null; runnerContainer?: string | null; createdAt: string; startedAt: string | null; finishedAt: string | null; exitCode: number | null; stdoutFile: string; stderrFile: string; note: string; } export interface JobProgressSummary { kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "agentrun-yaml-lane-trigger" | "git-mirror" | "generic"; stage: string | null; stageStatus: string | null; sourceCommit: string | null; pipelineRun: string | null; pipelineCreated: boolean | null; elapsedSeconds: number | null; stageElapsedSeconds: number | null; lastEventAt: string | null; lastEventAgeSeconds: number | null; eventsObserved: number; slow: boolean; warnings: string[]; diagnostics?: Record | null; timings: Record; summary: string; nextCommand: string | null; } export interface StartJobOptions { runner?: "local" | "docker"; dockerImage?: string; } function jobsDir(): string { const dir = rootPath(".state", "jobs"); mkdirSync(dir, { recursive: true }); return dir; } function jobPath(id: string): string { return join(jobsDir(), `${id}.json`); } function writeJob(job: JobRecord): void { writeFileSync(jobPath(job.id), `${JSON.stringify(job, null, 2)}\n`, "utf8"); } export function readJob(id: string): JobRecord { const path = jobPath(id); if (!existsSync(path)) throw new Error(`job not found: ${id}`); return JSON.parse(readFileSync(path, "utf8")) as JobRecord; } export function cancelJob(id: string): unknown { const job = readJob(id); if (job.status !== "queued" && job.status !== "running") { return { ok: true, action: "cancel-job", id, alreadyTerminal: true, job }; } const actions: Array> = []; if (job.runnerPid !== null && job.runnerPid !== undefined) { try { process.kill(-job.runnerPid, "SIGTERM"); actions.push({ signal: "SIGTERM", target: "process-group", pid: job.runnerPid, ok: true }); } catch (error) { actions.push({ signal: "SIGTERM", target: "process-group", pid: job.runnerPid, ok: false, error: error instanceof Error ? error.message : String(error) }); try { process.kill(job.runnerPid, "SIGTERM"); actions.push({ signal: "SIGTERM", target: "process", pid: job.runnerPid, ok: true }); } catch (innerError) { actions.push({ signal: "SIGTERM", target: "process", pid: job.runnerPid, ok: false, error: innerError instanceof Error ? innerError.message : String(innerError) }); } } } job.status = "canceled"; job.finishedAt = new Date().toISOString(); job.exitCode = 130; writeFileSync(job.stderrFile, `${tailFile(job.stderrFile, 512_000)}${JSON.stringify({ event: "unidesk.job.cancel", at: job.finishedAt, jobId: id, actions })}\n`, "utf8"); writeJob(job); return { ok: true, action: "cancel-job", id, actions, job }; } export function listJobs(): JobRecord[] { return readdirSync(jobsDir()) .filter((name) => name.endsWith(".json")) .map((name) => JSON.parse(readFileSync(join(jobsDir(), name), "utf8")) as JobRecord) .sort((a, b) => b.createdAt.localeCompare(a.createdAt)); } export function startJob(name: string, command: string[], note: string, options: StartJobOptions = {}): JobRecord { const id = `${name}_${new Date().toISOString().replace(/[-:.TZ]/g, "")}_${Math.random().toString(16).slice(2, 8)}`; const stdoutFile = rootPath(".state", "jobs", `${id}.stdout.log`); const stderrFile = rootPath(".state", "jobs", `${id}.stderr.log`); const runner = options.runner ?? "local"; const job: JobRecord = { id, name, status: "queued", command, cwd: repoRoot, runner, runnerPid: null, runnerContainer: null, createdAt: new Date().toISOString(), startedAt: null, finishedAt: null, exitCode: null, stdoutFile, stderrFile, note, }; writeJob(job); if (runner === "docker") { const containerName = `unidesk-job-runner-${id}`.replace(/[^A-Za-z0-9_.-]/g, "-").slice(0, 120); job.runnerContainer = containerName; writeJob(job); const dockerArgs = [ "run", "-d", "--rm", "--name", containerName, "-v", "/var/run/docker.sock:/var/run/docker.sock", "-v", `${repoRoot}:${repoRoot}`, "-w", repoRoot, "--entrypoint", "bun", options.dockerImage ?? "unidesk-code-queue:latest", rootPath("scripts", "cli.ts"), "internal", "run-job", id, ]; const result = spawnSync("docker", dockerArgs, { cwd: repoRoot, encoding: "utf8" }); if (result.status !== 0) { job.status = "failed"; job.startedAt = new Date().toISOString(); job.finishedAt = new Date().toISOString(); job.exitCode = result.status ?? 127; writeFileSync(stderrFile, result.stderr || result.error?.message || "failed to start docker job runner\n", "utf8"); writeFileSync(stdoutFile, result.stdout || "", "utf8"); writeJob(job); } return job; } const child = spawn(process.execPath, [rootPath("scripts", "cli.ts"), "internal", "run-job", id], { cwd: repoRoot, detached: true, stdio: "ignore", env: process.env, }); job.runnerPid = child.pid ?? null; writeJob(job); child.unref(); return job; } export async function runJob(id: string): Promise { const job = readJob(id); job.status = "running"; job.runnerPid = process.pid; job.startedAt = new Date().toISOString(); writeJob(job); const exitCode = await runCommandToFiles(job.command, job.cwd, job.stdoutFile, job.stderrFile); job.exitCode = exitCode; job.status = exitCode === 0 ? "succeeded" : "failed"; job.finishedAt = new Date().toISOString(); writeJob(job); return job; } export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & { progress: JobProgressSummary; tailPolicy: { requestedTailBytes: number; stdoutBytes: number; stderrBytes: number; stdoutTruncated: boolean; stderrTruncated: boolean; fullLogPaths: { stdoutFile: string; stderrFile: string }; }; stdoutTail: string; stderrTail: string; } { const stdoutBytes = existsSync(job.stdoutFile) ? statSync(job.stdoutFile).size : 0; const stderrBytes = existsSync(job.stderrFile) ? statSync(job.stderrFile).size : 0; const progressTailBytes = Math.max(maxBytes, 96_000); const stdoutProgressTail = tailFile(job.stdoutFile, progressTailBytes); const stderrProgressTail = tailFile(job.stderrFile, progressTailBytes); const progress = summarizeJobProgress(job, progressTailBytes, { stdoutTail: stdoutProgressTail, stderrTail: stderrProgressTail }); const rawStdoutTail = tailTextByBytes(stdoutProgressTail, maxBytes); const stdoutTail = compactJobStdoutTail(job, progress, rawStdoutTail, maxBytes); return { ...job, progress, tailPolicy: { requestedTailBytes: maxBytes, stdoutBytes, stderrBytes, stdoutTruncated: stdoutBytes > maxBytes, stderrTruncated: stderrBytes > maxBytes, stdoutCompacted: stdoutTail !== rawStdoutTail, fullLogPaths: { stdoutFile: job.stdoutFile, stderrFile: job.stderrFile }, }, stdoutTail, stderrTail: tailTextByBytes(stderrProgressTail, maxBytes), }; } export function renderJobStatusSummary(job: ReturnType): RenderedCliResult { const progress = job.progress; const warnings = Array.isArray(progress.warnings) ? progress.warnings : []; const lines = [ jobStatusTable( ["JOB", "STATUS", "EXIT", "KIND", "STAGE", "STAGE_STATUS", "ELAPSED", "LAST_EVENT_AGE"], [[job.id, job.status, job.exitCode ?? "-", progress.kind, progress.stage ?? "-", progress.stageStatus ?? "-", secondsText(progress.elapsedSeconds), secondsText(progress.lastEventAgeSeconds)]], ), "", jobStatusTable( ["SOURCE", "PIPELINE", "CREATED", "EVENTS", "SLOW", "SUMMARY"], [[progress.sourceCommit ? String(progress.sourceCommit).slice(0, 12) : "-", progress.pipelineRun ?? "-", progress.pipelineCreated ?? "-", progress.eventsObserved, progress.slow, progress.summary]], ), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.slice(0, 6).map((item) => `- ${jobStatusCell(item, 220)}`)].join("\n"), "", jobStatusTable( ["TAIL", "BYTES", "TRUNCATED", "PATH"], [ ["stdout", job.tailPolicy.stdoutBytes, job.tailPolicy.stdoutTruncated, job.tailPolicy.fullLogPaths.stdoutFile], ["stderr", job.tailPolicy.stderrBytes, job.tailPolicy.stderrTruncated, job.tailPolicy.fullLogPaths.stderrFile], ], ), "", jobTailSection("STDOUT_TAIL", job.stdoutTail), "", jobTailSection("STDERR_TAIL", job.stderrTail), "", "NEXT", ` status: bun scripts/cli.ts job status ${job.id} --tail-bytes ${job.tailPolicy.requestedTailBytes}`, progress.nextCommand ? ` progress: ${progress.nextCommand}` : null, ` full: bun scripts/cli.ts job status ${job.id} --tail-bytes ${job.tailPolicy.requestedTailBytes} --full`, ].filter((line): line is string => line !== null); return { ok: job.status !== "failed", command: "job status", contentType: "text/plain", renderedText: `${lines.join("\n")}\n`, }; } export function renderJobLaunchSummary(command: string, result: unknown): RenderedCliResult | null { if (typeof result !== "object" || result === null || !("job" in result)) return null; const record = result as Record; const job = record.job as Partial | undefined; if (typeof job !== "object" || job === null || typeof job.id !== "string") return null; const service = typeof record.service === "string" ? record.service : "-"; const runtimeEnv = typeof record.runtimeEnv === "object" && record.runtimeEnv !== null ? record.runtimeEnv as Record : {}; const stdoutFile = typeof job.stdoutFile === "string" ? job.stdoutFile : "-"; const stderrFile = typeof job.stderrFile === "string" ? job.stderrFile : "-"; const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`; const fullCommand = command.includes("--full") || command.includes("--raw") ? null : `bun scripts/cli.ts ${command} --full`; const lines = [ "ASYNC_JOB", jobStatusTable( ["JOB", "NAME", "STATUS", "SERVICE", "RUNNER", "CREATED"], [[job.id, job.name ?? "-", job.status ?? "-", service, job.runner ?? "-", job.createdAt ?? "-"]], ), "", jobStatusTable( ["STREAM", "PATH"], [ ["stdout", stdoutFile], ["stderr", stderrFile], ["runtimeEnv", typeof runtimeEnv.envFile === "string" ? runtimeEnv.envFile : "-"], ["logDir", typeof runtimeEnv.logDir === "string" ? runtimeEnv.logDir : "-"], ], ), "", "NEXT", ` status: ${statusCommand}`, " latest: bun scripts/cli.ts job status latest --tail-bytes 12000", " list: bun scripts/cli.ts job list --limit 20", " logs: bun scripts/cli.ts server logs --tail-bytes 3000", " server: bun scripts/cli.ts server status", fullCommand === null ? null : ` full: ${fullCommand}`, ].filter((line): line is string => line !== null); return { ok: job.status !== "failed", command, contentType: "text/plain", renderedText: `${lines.join("\n")}\n`, }; } function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary { const nowMs = Date.now(); const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current"; const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor"; const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name); const agentRunYamlLaneTriggerWorkflow = /^agentrun_v[0-9]{2}_trigger_current$/u.test(job.name); const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush" || /^hwlab_nodes_v[0-9]{2}_git-mirror_(sync|flush)$/u.test(job.name); const ciInstallWorkflow = job.name === "ci_install"; if (ciInstallWorkflow) return summarizeCiInstallJobProgress(job, tails?.stderrTail, nowMs); const progressTailBytes = Math.max(4096, Math.floor(maxBytes)); const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes); const agentRunYamlLaneProgressObserved = agentRunYamlLaneTriggerWorkflow || hasAgentRunYamlLaneProgressEvents(stderrTail); if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !agentRunYamlLaneProgressObserved && !gitMirrorWorkflow) return genericJobProgress(job, stderrTail); const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes); if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs); if (runtimeLaneTriggerWorkflow) return summarizeRuntimeLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs); if (agentRunYamlLaneProgressObserved) return summarizeAgentRunYamlLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs); const events = v02PrMonitorWorkflow ? [ ...parseJsonLineEvents(stderrTail, "hwlab.v02.pr-monitor.progress"), ...parseJsonLineEvents(stderrTail, "hwlab.v02.trigger.progress"), ].sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? ""))) : parseJsonLineEvents(stderrTail, "hwlab.v02.trigger.progress"); const lastEvent = events.at(-1) ?? {}; const stage = stringField(lastEvent.stage); const stageStatus = stringField(lastEvent.status); const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu); const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u); const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail) ? true : stage === "create-pipelinerun" && stageStatus === "failed" ? false : null; const lastEventAt = stringField(lastEvent.at); const kind = events.length > 0 || knownWorkflow || v02PrMonitorWorkflow ? "hwlab-v02-trigger" : "generic"; const elapsedSeconds = jobElapsedSeconds(job, nowMs); const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs); const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs); const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`); const warnings = jobProgressWarnings({ job, eventsObserved: events.length, elapsedSeconds, stage, stageStatus, stageElapsedSeconds, lastEventAgeSeconds, }); const slow = warnings.length > 0; const nextCommand = pipelineRun ? `bun scripts/cli.ts hwlab g14 control-plane status --lane v02` : job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null; const summary = kind === "hwlab-v02-trigger" ? [ job.status, stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown", sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null, pipelineRun ? `pipelineRun=${pipelineRun}` : null, pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null, elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null, lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null, slow ? "visibility-warning" : null, ].filter(Boolean).join(" ") : `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`; return { kind, stage, stageStatus, sourceCommit, pipelineRun, pipelineCreated, elapsedSeconds, stageElapsedSeconds, lastEventAt, lastEventAgeSeconds, eventsObserved: events.length, slow, warnings, timings, summary, nextCommand, }; } function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary { const action = job.name.endsWith("_flush") ? "flush" : job.name.endsWith("_sync") ? "sync" : "unknown"; const elapsedSeconds = jobElapsedSeconds(job, nowMs); const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`); const status = firstMatch(stdoutTail, new RegExp(`"event"\\s*:\\s*"git-mirror-${action}"[\\s\\S]{0,240}?"status"\\s*:\\s*"([^"]+)"`, "u")) ?? firstMatch(stdoutTail.replaceAll('\\"', '"').replaceAll("\\n", "\n"), new RegExp(`"event"\\s*:\\s*"git-mirror-${action}"[\\s\\S]{0,240}?"status"\\s*:\\s*"([^"]+)"`, "u")); const pendingFlush = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"pendingFlush"\s*:\s*(true|false)/u); const partialSuccess = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"partialSuccess"\s*:\s*"([^"]+)"/u); const jobName = firstMatch(stdoutTail, /"jobName"\s*:\s*"([^"]+)"/u); const node = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"node"\s*:\s*"([^"]+)"/u); const lane = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"lane"\s*:\s*"(v[0-9]+)"/u) ?? firstMatch(job.name, /^hwlab_nodes_(v[0-9]+)_/u); const warnings = jobProgressWarnings({ job, eventsObserved: 0, elapsedSeconds, stage: action, stageStatus: job.status === "running" ? "running" : status, stageElapsedSeconds: job.status === "running" ? elapsedSeconds : null, lastEventAgeSeconds: null, }); const timingSummary = Object.entries(timings) .filter(([key]) => key.startsWith(`gitMirror.${action}.`) || key === "sshRuntimeMaxMs") .map(([key, value]) => `${key}=${value}ms`); return { kind: "git-mirror", stage: action, stageStatus: status ?? (job.status === "running" ? "running" : null), sourceCommit: null, pipelineRun: null, pipelineCreated: null, elapsedSeconds, stageElapsedSeconds: job.status === "running" ? elapsedSeconds : null, lastEventAt: null, lastEventAgeSeconds: null, eventsObserved: 0, slow: warnings.length > 0, warnings, timings, summary: [ job.status, `git-mirror-${action}${status ? `:${status}` : ""}`, partialSuccess ? `partialSuccess=${partialSuccess}` : null, jobName ? `job=${jobName}` : null, pendingFlush !== null ? `pendingFlush=${pendingFlush}` : null, elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, ...timingSummary, warnings.length > 0 ? "visibility-warning" : null, ].filter(Boolean).join(" "), nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : job.name.startsWith("agentrun_") ? "bun scripts/cli.ts agentrun git-mirror status" : job.name.startsWith("hwlab_nodes_") && node !== null && lane !== null ? partialSuccess === "push-succeeded-fetch-failed" ? `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm --wait` : `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}` : "bun scripts/cli.ts hwlab g14 git-mirror status", }; } function summarizeCiInstallJobProgress(job: JobRecord, stderrTailOverride?: string, nowMs = Date.now()): JobProgressSummary { const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000); const events = parseJsonLineEvents(stderrTail, "ci.install.progress") .sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? ""))); const lastEvent = events.at(-1) ?? {}; const stage = stringField(lastEvent.stage); const stageStatus = stringField(lastEvent.status); const lastEventAt = stringField(lastEvent.at); const elapsedSeconds = jobElapsedSeconds(job, nowMs); const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs); const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs); const warnings = jobProgressWarnings({ job, eventsObserved: events.length, elapsedSeconds, stage, stageStatus, stageElapsedSeconds, lastEventAgeSeconds, }); return { kind: "generic", stage, stageStatus, sourceCommit: null, pipelineRun: null, pipelineCreated: null, elapsedSeconds, stageElapsedSeconds, lastEventAt, lastEventAgeSeconds, eventsObserved: events.length, slow: warnings.length > 0, warnings, timings: {}, summary: [ job.status, stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown", typeof lastEvent.providerId === "string" ? `provider=${lastEvent.providerId}` : null, typeof lastEvent.manifest === "string" ? `manifest=${lastEvent.manifest}` : null, elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null, lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null, warnings.length > 0 ? "visibility-warning" : null, ].filter(Boolean).join(" "), nextCommand: job.status === "running" ? `bun scripts/cli.ts ci install-status ${job.id}` : "bun scripts/cli.ts ci status", }; } function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary { const events = parseJsonLineEvents(stderrTail, "hwlab.runtime-lane.trigger.progress") .sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? ""))); const lastEvent = events.at(-1) ?? {}; const stage = stringField(lastEvent.stage); const stageStatus = stringField(lastEvent.status); const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu); const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u); const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail) ? true : stage === "create-pipelinerun" && stageStatus === "failed" ? false : null; const lastEventAt = stringField(lastEvent.at); const elapsedSeconds = jobElapsedSeconds(job, nowMs); const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs); const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs); const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`); const warnings = jobProgressWarnings({ job, eventsObserved: events.length, elapsedSeconds, stage, stageStatus, stageElapsedSeconds, lastEventAgeSeconds, }); const nextStatusCommand = hwlabRuntimeLaneStatusNextCommand(pipelineRun, lastEvent); if (nextStatusCommand.warning !== null) warnings.push(nextStatusCommand.warning); const tcpPoolDiagnostics = job.status === "succeeded" ? null : sshTcpPoolDiagnosticsFromJobText(`${stderrTail}\n${stdoutTail}`); const pipelineFailureDiagnostics = runtimeLanePipelineFailureDiagnostics(stdoutTail); const slow = warnings.length > 0; const diagnostics = pipelineFailureDiagnostics !== null ? { pipelineFailure: pipelineFailureDiagnostics, sshTcpPool: tcpPoolDiagnostics } : tcpPoolDiagnostics; return { kind: "hwlab-runtime-lane-trigger", stage, stageStatus, sourceCommit, pipelineRun, pipelineCreated, elapsedSeconds, stageElapsedSeconds, lastEventAt, lastEventAgeSeconds, eventsObserved: events.length, slow, warnings, diagnostics, timings, summary: [ job.status, stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown", sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null, pipelineRun ? `pipelineRun=${pipelineRun}` : null, pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null, pipelineFailureDiagnostics !== null ? runtimeLanePipelineFailureSummaryText(pipelineFailureDiagnostics) : null, elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null, lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null, tcpPoolDiagnostics !== null ? `sshPool=${String(tcpPoolDiagnostics.failureKind ?? "transient")}` : null, slow ? "visibility-warning" : null, ].filter(Boolean).join(" "), nextCommand: pipelineRun ? nextStatusCommand.command : job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null, }; } const agentRunYamlLaneProgressEvents = [ { event: "agentrun.yaml-lane.source-bootstrap.progress", stage: "source-bootstrap" }, { event: "agentrun.yaml-lane.image-build.progress", stage: "image-build" }, { event: "agentrun.yaml-lane.gitops-publish.progress", stage: "gitops-publish" }, { event: "agentrun.yaml-lane.git-mirror.progress", stage: "git-mirror" }, ] as const; function hasAgentRunYamlLaneProgressEvents(text: string): boolean { return parseAgentRunYamlLaneProgressEvents(text).length > 0; } function summarizeAgentRunYamlLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary { const events = parseAgentRunYamlLaneProgressEvents(stderrTail); const lastEvent = events.at(-1) ?? {}; const stage = stringField(lastEvent.stage); const stageStatus = agentRunYamlLaneStageStatus(lastEvent); const sourceCommit = lastStringField(events, "sourceCommit") ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu); const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u); const pipelineCreatedValue = firstMatch(stdoutTail, /"created"\s*:\s*\{[\s\S]{0,400}?"created"\s*:\s*(true|false)/u); const pipelineCreated = pipelineCreatedValue === "true" ? true : pipelineCreatedValue === "false" ? false : null; const lastEventAt = stringField(lastEvent.at); const elapsedSeconds = jobElapsedSeconds(job, nowMs); const eventElapsedMs = numberField(lastEvent.elapsedMs); const stageElapsedSeconds = eventElapsedMs === null ? null : Math.round(eventElapsedMs / 1000); const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs); const timings = agentRunYamlLaneTimings(events); const warnings = jobProgressWarnings({ job, eventsObserved: events.length, elapsedSeconds, stage, stageStatus, stageElapsedSeconds, lastEventAgeSeconds, }); const slow = warnings.length > 0; const eventNode = stringField(lastEvent.node); const eventLane = stringField(lastEvent.lane) ?? firstMatch(job.name, /^agentrun_(v[0-9]{2})_trigger_current$/u); const nextCommand = pipelineRun ? [ "bun scripts/cli.ts agentrun control-plane status", eventNode ? `--node ${eventNode}` : null, eventLane ? `--lane ${eventLane}` : null, `--pipeline-run ${pipelineRun}`, ].filter(Boolean).join(" ") : job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null; return { kind: "agentrun-yaml-lane-trigger", stage, stageStatus, sourceCommit, pipelineRun, pipelineCreated, elapsedSeconds, stageElapsedSeconds, lastEventAt, lastEventAgeSeconds, eventsObserved: events.length, slow, warnings, diagnostics: { stages: agentRunYamlLaneStageDiagnostics(events), }, timings, summary: [ job.status, stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown", sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null, pipelineRun ? `pipelineRun=${pipelineRun}` : null, pipelineCreated === true ? "created" : pipelineCreated === false ? "create-reused" : null, elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null, lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null, `events=${events.length}`, slow ? "visibility-warning" : null, ].filter(Boolean).join(" "), nextCommand, }; } function parseAgentRunYamlLaneProgressEvents(text: string): Record[] { const eventToStage = new Map(agentRunYamlLaneProgressEvents.map((item) => [item.event, item.stage])); const events: Record[] = []; const lines = text.split(/\r?\n/u); for (let index = 0; index < lines.length; index += 1) { const line = lines[index]?.trim() ?? ""; if (line.length === 0) continue; if (line.startsWith("{")) { try { const parsed = JSON.parse(line) as unknown; if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) { const record = parsed as Record; const event = typeof record.event === "string" ? record.event : null; const stage = event === null ? null : eventToStage.get(event) ?? null; if (event !== null && stage !== null) events.push({ ...record, stage, __order: index }); } } catch { // Ignore non-JSON progress noise; raw tails remain visible. } continue; } const match = line.match(/^status\s+(agentrun\.yaml-lane\.[a-z-]+\.progress)\s+([a-z-]+)(?:\s+(.*))?$/u); if (!match) continue; const event = match[1]!; const stage = eventToStage.get(event); if (stage === undefined) continue; const payload: Record = { event, stage, status: match[2]!, __order: index }; for (const token of (match[3] ?? "").split(/\s+/u).filter(Boolean)) { const tokenMatch = token.match(/^([A-Za-z][A-Za-z0-9_]*)=(.+)$/u); if (!tokenMatch) continue; const key = tokenMatch[1]!; const value = tokenMatch[2]!; payload[key] = /^-?\d+$/u.test(value) ? Number(value) : value; } events.push(payload); } return events.sort((left, right) => Number(left.__order ?? 0) - Number(right.__order ?? 0)); } function agentRunYamlLaneStageStatus(event: Record): string | null { const explicitStatus = stringField(event.status); if (explicitStatus !== null) return explicitStatus; if (event.succeeded === true) return "succeeded"; if (event.failed === true) return "failed"; if (event.stage !== undefined) return "running"; return null; } function agentRunYamlLaneTimings(events: Record[]): Record { const timings: Record = {}; for (const event of events) { const stage = stringField(event.stage); const elapsedMs = numberField(event.elapsedMs); if (stage !== null && elapsedMs !== null) timings[`agentrunYamlLane.${stage}.elapsedMs`] = Math.round(elapsedMs); } return timings; } function agentRunYamlLaneStageDiagnostics(events: Record[]): Array> { return agentRunYamlLaneProgressEvents .map(({ stage }) => { const stageEvents = events.filter((event) => stringField(event.stage) === stage); const lastEvent = stageEvents.at(-1) ?? {}; return { stage, eventsObserved: stageEvents.length, status: agentRunYamlLaneStageStatus(lastEvent), jobId: stringField(lastEvent.jobId), jobName: stringField(lastEvent.jobName), polls: numberField(lastEvent.polls), elapsedMs: numberField(lastEvent.elapsedMs), }; }) .filter((item) => item.eventsObserved > 0); } function hwlabRuntimeLaneStatusNextCommand(pipelineRun: string | null, lastEvent: Record): { command: string | null; warning: string | null } { if (pipelineRun === null) return { command: null, warning: null }; const eventNode = stringField(lastEvent.node); const eventLane = stringField(lastEvent.lane); const defaults = hwlabDefaultRuntimeTarget(); const node = eventNode ?? defaults.node; const lane = eventLane ?? defaults.lane; const missing = [ eventNode === null ? "node" : null, eventLane === null ? "lane" : null, ].filter((value): value is string => value !== null); return { command: `bun scripts/cli.ts hwlab nodes control-plane status --node ${node} --lane ${lane} --pipeline-run ${pipelineRun}`, warning: missing.length === 0 ? null : `runtime lane progress event missing ${missing.join("/")} for nextCommand; used YAML default ${defaults.node}/${defaults.lane} from config/hwlab-node-lanes.yaml`, }; } function sshTcpPoolDiagnosticsFromJobText(text: string): Record | null { const hint = lastJsonLinePayload(text, "UNIDESK_SSH_TCP_POOL_HINT"); if (hint !== null) return hint; const error = lastJsonLinePayload(text, "UNIDESK_SSH_ERROR"); if (error !== null && typeof error.failureKind === "string" && error.failureKind.startsWith("provider-data-")) { return { code: "ssh-tcp-pool-transient", failureKind: error.failureKind, providerId: error.providerId ?? null, dataChannelId: error.dataChannelId ?? null, dataPool: error.dataPool ?? null, diagnostics: { fullHealth: "bun scripts/cli.ts debug health" }, }; } if (/ssh tcp data channel closed/iu.test(text)) { return { code: "ssh-tcp-pool-transient", failureKind: "provider-data-channel-closed", diagnostics: { fullHealth: "bun scripts/cli.ts debug health" } }; } if (/requested ssh tcp data channel is not ready|ssh tcp data channel is not available/iu.test(text)) { return { code: "ssh-tcp-pool-transient", failureKind: "provider-data-channel-missing", diagnostics: { fullHealth: "bun scripts/cli.ts debug health" } }; } if (/provider ssh tcp data pool has no idle channel/iu.test(text)) { return { code: "ssh-tcp-pool-transient", failureKind: "provider-data-pool-exhausted", diagnostics: { fullHealth: "bun scripts/cli.ts debug health" } }; } return null; } function lastJsonLinePayload(text: string, prefix: string): Record | null { const lines = text.split(/\r?\n/u); for (let index = lines.length - 1; index >= 0; index -= 1) { const line = lines[index]?.trim() ?? ""; if (!line.startsWith(`${prefix} `)) continue; try { const parsed = JSON.parse(line.slice(prefix.length + 1)) as unknown; if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) return parsed as Record; } catch { return null; } } return null; } function runtimeLanePipelineFailureDiagnostics(stdoutTail: string): Record | null { const parsed = parseJsonObjectFromText(stdoutTail); const data = recordField(parsed.data); const directSummary = recordField(data.pipelineFailureSummary); const waitSummary = recordField(recordField(data.pipelineWait).failureSummary); const statusSummary = recordField(recordField(recordField(data.pipelineRun).diagnostics).failureSummary); const failureSummary = Object.keys(directSummary).length > 0 ? directSummary : Object.keys(waitSummary).length > 0 ? waitSummary : statusSummary; const firstFailure = recordField(failureSummary.firstFailure); if (Object.keys(failureSummary).length > 0) { return { degradedReason: data.degradedReason ?? null, failedTaskRunCount: failureSummary.failedTaskRunCount ?? null, failedStepCount: failureSummary.failedStepCount ?? null, stepPublishFailureCount: failureSummary.stepPublishFailureCount ?? null, firstFailure: compactRuntimeLaneFailure(firstFailure), nextAction: failureSummary.nextAction ?? null, }; } const taskRun = firstMatch(stdoutTail, /"taskRun"\s*:\s*"([^"]+)"/u); const pod = firstMatch(stdoutTail, /"pod"\s*:\s*"([^"]+)"/u); const step = firstMatch(stdoutTail, /"step"\s*:\s*"([^"]+)"/u); const container = firstMatch(stdoutTail, /"container"\s*:\s*"([^"]+)"/u); if (taskRun === null && pod === null && step === null && container === null) return null; return { degradedReason: firstMatch(stdoutTail, /"degradedReason"\s*:\s*"([^"]+)"/u), failedTaskRunCount: null, failedStepCount: null, stepPublishFailureCount: container === "step-publish" || step === "publish" || step === "step-publish" ? 1 : 0, firstFailure: { taskRun, pod, step, container, logCommand: firstMatch(stdoutTail, /"logCommand"\s*:\s*"([^"]+)"/u) }, nextAction: null, }; } function runtimeLanePipelineFailureSummaryText(diagnostics: Record): string | null { const firstFailure = recordField(diagnostics.firstFailure); const taskRun = stringField(firstFailure.taskRun); const step = stringField(firstFailure.step); const container = stringField(firstFailure.container); const stepPublishCount = numberField(diagnostics.stepPublishFailureCount); const failedTaskRunCount = numberField(diagnostics.failedTaskRunCount); const parts = [ failedTaskRunCount !== null ? `failedTaskRuns=${failedTaskRunCount}` : null, taskRun !== null ? `taskRun=${taskRun}` : null, step !== null ? `step=${step}` : null, container !== null ? `container=${container}` : null, stepPublishCount !== null && stepPublishCount > 0 ? "step-publish" : null, ]; const text = parts.filter(Boolean).join(" "); return text.length > 0 ? text : null; } function compactRuntimeLaneFailure(value: Record): Record | null { if (Object.keys(value).length === 0) return null; return { taskRun: value.taskRun ?? null, pipelineTask: value.pipelineTask ?? null, taskRef: value.taskRef ?? null, pod: value.pod ?? null, step: value.step ?? null, container: value.container ?? null, terminationReason: value.terminationReason ?? null, exitCode: value.exitCode ?? null, logCommand: value.logCommand ?? null, taskRunCommand: value.taskRunCommand ?? null, podDescribeCommand: value.podDescribeCommand ?? null, }; } function parseJsonObjectFromText(text: string): Record { const trimmed = text.trim(); if (trimmed.length === 0) return {}; try { return recordField(JSON.parse(trimmed) as unknown); } catch { const start = trimmed.indexOf("{"); const end = trimmed.lastIndexOf("}"); if (start >= 0 && end > start) { try { return recordField(JSON.parse(trimmed.slice(start, end + 1)) as unknown); } catch {} } } return {}; } function recordField(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary { const nowMs = Date.now(); const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000); const downloadProgressEvents = parseJsonLineEvents(stderrTail, "unidesk.ssh.download.progress"); const downloadRetryEvents = [ ...parseJsonLineEvents(stderrTail, "unidesk.ssh.download.empty-read-retry"), ...parseJsonLineEvents(stderrTail, "unidesk.ssh.download.short-read-retry"), ]; const downloadEvents = [...downloadProgressEvents, ...downloadRetryEvents].sort((left, right) => Number(left.blockIndex ?? -1) - Number(right.blockIndex ?? -1)); const lastDownload = downloadEvents.at(-1) ?? null; const lastDownloadAt = stringField(lastDownload?.at); const lastEventAgeSeconds = lastDownloadAt === null ? null : secondsSince(lastDownloadAt, job.finishedAt ?? nowMs); const elapsedSeconds = jobElapsedSeconds(job, nowMs); const warnings = jobProgressWarnings({ job, eventsObserved: downloadEvents.length, elapsedSeconds, stage: lastDownload === null ? null : "ssh-download", stageStatus: lastDownload === null ? null : "running", stageElapsedSeconds: null, lastEventAgeSeconds, }); const downloadSummary = lastDownload === null ? null : [ ` ssh-download ${Number(lastDownload.actualBytes ?? lastDownload.bytes ?? 0)}/${Number(lastDownload.expectedBytes ?? lastDownload.totalBytes ?? 0)} bytes`, `chunks=${Number(lastDownload.chunks ?? 0)}`, `remaining=${Number(lastDownload.remainingBytes ?? 0)}`, `throughput=${Number(lastDownload.throughputBytesPerSecond ?? 0)}Bps`, `elapsedMs=${Number(lastDownload.elapsedMs ?? 0)}`, downloadRetryEvents.length === 0 ? null : `retries=${downloadRetryEvents.length} lastRetry=${String(lastDownload.reason ?? "unknown")}`, ].filter(Boolean).join(" "); return { kind: "generic", stage: lastDownload === null ? null : "ssh-download", stageStatus: lastDownload === null ? null : "running", sourceCommit: null, pipelineRun: null, pipelineCreated: null, elapsedSeconds, stageElapsedSeconds: null, lastEventAt: lastDownloadAt, lastEventAgeSeconds, eventsObserved: downloadEvents.length, slow: warnings.length > 0, warnings, timings: {}, summary: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}${downloadSummary ?? ""}`, nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null, }; } function jobElapsedSeconds(job: JobRecord, nowMs = Date.now()): number | null { const startMs = timestampMs(job.startedAt ?? job.createdAt); if (startMs === null) return null; const endMs = timestampMs(job.finishedAt) ?? nowMs; if (endMs < startMs) return null; return Math.round((endMs - startMs) / 1000); } function secondsSince(value: string, end: string | number = Date.now()): number | null { const startMs = timestampMs(value); const endMs = typeof end === "number" ? end : timestampMs(end); if (startMs === null || endMs === null || endMs < startMs) return null; return Math.round((endMs - startMs) / 1000); } function timestampMs(value: unknown): number | null { if (typeof value !== "string" || value.trim().length === 0) return null; const parsed = Date.parse(value); return Number.isFinite(parsed) ? parsed : null; } function currentStageElapsedSeconds(events: Record[], stage: string | null, stageStatus: string | null, job: JobRecord, nowMs = Date.now()): number | null { if (stage === null) return null; const stageEvents = events.filter((event) => stringField(event.stage) === stage); const startEvent = stageEvents.findLast((event) => stringField(event.status) === "started"); const startAt = stringField(startEvent?.at); if (startAt === null) return null; const endEvent = stageStatus === "started" ? null : stageEvents.findLast((event) => { const status = stringField(event.status); return status !== null && status !== "started"; }); const endAt = stringField(endEvent?.at) ?? job.finishedAt ?? (job.status === "running" ? new Date(nowMs).toISOString() : null); return endAt === null ? null : secondsSince(startAt, endAt); } function jobProgressWarnings(input: { job: JobRecord; eventsObserved: number; elapsedSeconds: number | null; stage: string | null; stageStatus: string | null; stageElapsedSeconds: number | null; lastEventAgeSeconds: number | null; }): string[] { if (input.job.status !== "running") return []; if (input.stage === "idle" && input.stageStatus === "waiting") return []; const warnings: string[] = []; if (input.eventsObserved === 0 && (input.elapsedSeconds ?? 0) >= 30) { warnings.push(`running for ${input.elapsedSeconds}s without progress events; inspect stdoutTail/stderrTail and fullLogPaths`); } if ((input.lastEventAgeSeconds ?? 0) >= 30) { warnings.push(`no progress event for ${input.lastEventAgeSeconds}s; current stage may be blocked or waiting on remote I/O`); } if (input.stage !== null && input.stageStatus !== null && input.stageStatus !== "succeeded" && input.stageStatus !== "failed" && (input.stageElapsedSeconds ?? 0) >= 30) { warnings.push(`stage ${input.stage}:${input.stageStatus} has been active for ${input.stageElapsedSeconds}s`); } return warnings; } function tailTextByBytes(text: string, maxBytes: number): string { const safeMaxBytes = Math.max(0, Math.floor(maxBytes)); if (safeMaxBytes === 0) return ""; const buffer = Buffer.from(text, "utf8"); if (buffer.length <= safeMaxBytes) return text; return buffer.subarray(buffer.length - safeMaxBytes).toString("utf8"); } function secondsText(value: number | null): string { return value === null ? "-" : `${value}s`; } function jobStatusCell(value: unknown, maxLength = 96): string { if (value === null || value === undefined) return "-"; const text = typeof value === "string" ? value : String(value); const compact = text.replace(/\s+/gu, " ").trim(); if (compact.length === 0) return "-"; if (compact.length <= maxLength) return compact; return `${compact.slice(0, Math.max(1, maxLength - 1))}...`; } function jobStatusTable(headers: string[], rows: unknown[][]): string { const stringRows = rows.map((row) => row.map((value) => jobStatusCell(value))); const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0))); const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd(); return [renderRow(headers), ...stringRows.map(renderRow)].join("\n"); } function jobTailSection(title: string, text: string): string { const lines = text.trim().split(/\r?\n/u).filter((line) => line.trim().length > 0).slice(-6).map((line) => ` ${jobStatusCell(line, 220)}`); return lines.length === 0 ? `${title}\n-` : [title, ...lines].join("\n"); } function compactJobStdoutTail(job: JobRecord, progress: JobProgressSummary, rawTail: string, requestedTailBytes: number): string { if (progress.kind !== "hwlab-runtime-lane-trigger") return rawTail; if (Buffer.byteLength(rawTail, "utf8") <= 4096 && !rawTail.includes("\"expected\"") && !rawTail.includes("stdoutTail")) return rawTail; const interestingLines = rawTail.split(/\r?\n/u) .map((line) => line.trimEnd()) .filter((line) => !/"(stdoutTail|stderrTail|expected|result)"\s*:/u.test(line)) .filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|pipelineFailureSummary|failureSummary|firstFailure|failedTaskRun|stepFailure|taskRun|pod|step|container|terminationReason|exitCode|logCommand|taskRunCommand|podDescribeCommand|step-publish|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line)) .slice(-80); const header = [ `[stdout compacted: ${progress.kind}; requestedTailBytes=${requestedTailBytes}]`, `full stdout: ${job.stdoutFile}`, ].join("\n"); if (interestingLines.length === 0) return `${header}\n${tailTextByBytes(rawTail, Math.min(requestedTailBytes, 3000))}`; return `${header}\n${interestingLines.join("\n")}\n`; } function parseJsonLineEvents(text: string, eventName: string): Record[] { const events: Record[] = []; for (const line of text.split(/\r?\n/u)) { const trimmed = line.trim(); if (!trimmed.startsWith("{")) continue; try { const parsed = JSON.parse(trimmed) as unknown; if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) && (parsed as Record).event === eventName) { events.push(parsed as Record); } } catch { // Ignore non-JSON stderr lines; the raw tail remains available in stderrTail. } } return events; } function extractGitMirrorTimings(text: string): Record { const timings: Record = {}; for (const normalizedText of [text, text.replaceAll('\\"', '"').replaceAll("\\n", "\n")]) { extractGitMirrorTimingJsonLines(normalizedText, timings); extractGitMirrorTimingRegex(normalizedText, timings); extractSshRuntimeTimingRegex(normalizedText, timings); } return timings; } function extractGitMirrorTimingJsonLines(text: string, timings: Record): void { for (const line of text.split(/\r?\n/u)) { const trimmed = line.trim(); if (!trimmed.startsWith("{")) continue; try { const parsed = JSON.parse(trimmed) as unknown; if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) continue; const event = parsed as Record; const operation = event.event === "git-mirror-sync-timing" ? "sync" : event.event === "git-mirror-flush-timing" ? "flush" : null; if (operation === null) continue; const phase = stringField(event.phase); const durationMs = typeof event.durationMs === "number" && Number.isFinite(event.durationMs) ? event.durationMs : null; if (phase !== null && durationMs !== null) { timings[`gitMirror.${operation}.${phase}Ms`] = Math.round(durationMs); if (operation === "sync") timings[`gitMirror.${phase}Ms`] = Math.round(durationMs); } } catch { // Ignore non-JSON lines; raw logs remain available through job status tails. } } } function extractGitMirrorTimingRegex(text: string, timings: Record): void { const pattern = /"event"\s*:\s*"git-mirror-(sync|flush)-timing"[\s\S]{0,240}?"phase"\s*:\s*"([^"]+)"[\s\S]{0,240}?"durationMs"\s*:\s*(\d+)/gu; for (const match of text.matchAll(pattern)) { const operation = match[1]; const phase = match[2]; const durationMs = Number(match[3]); if (operation !== undefined && phase !== undefined && Number.isFinite(durationMs)) { timings[`gitMirror.${operation}.${phase}Ms`] = Math.round(durationMs); if (operation === "sync") timings[`gitMirror.${phase}Ms`] = Math.round(durationMs); } } } function extractSshRuntimeTimingRegex(text: string, timings: Record): void { const pattern = /"code"\s*:\s*"ssh-runtime-timing"[\s\S]{0,320}?"elapsedMs"\s*:\s*(\d+)/gu; const values = [...text.matchAll(pattern)] .map((match) => Number(match[1])) .filter((value) => Number.isFinite(value)); if (values.length > 0) timings.sshRuntimeMaxMs = Math.max(...values); } function stringField(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } function numberField(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim().length > 0) { const parsed = Number(value); if (Number.isFinite(parsed)) return parsed; } return null; } function lastStringField(events: Record[], field: string): string | null { const event = events.findLast((item) => stringField(item[field]) !== null); return stringField(event?.[field]); } function firstMatch(text: string, pattern: RegExp): string | null { const match = pattern.exec(text); return typeof match?.[1] === "string" && match[1].length > 0 ? match[1] : null; } export interface JobListOptions { limit?: number; includeCommand?: boolean; } export function listJobsSummary(options: JobListOptions = {}): unknown { const limit = Math.max(1, Math.floor(options.limit ?? 50)); const jobs = listJobs(); const returned = jobs.slice(0, limit).map((job) => ({ id: job.id, name: job.name, status: job.status, progress: summarizeJobProgress(job, 32_000), runner: job.runner, runnerPid: job.runnerPid ?? null, runnerContainer: job.runnerContainer ?? null, createdAt: job.createdAt, startedAt: job.startedAt, finishedAt: job.finishedAt, exitCode: job.exitCode, note: job.note, stdoutFile: job.stdoutFile, stderrFile: job.stderrFile, ...(options.includeCommand === true ? { command: job.command, cwd: job.cwd } : {}), })); return { jobs: returned, total: jobs.length, returned: returned.length, limit, truncated: jobs.length > returned.length, disclosure: { defaultLimit: 50, nextCommand: jobs.length > returned.length ? `bun scripts/cli.ts job list --limit ${Math.min(jobs.length, limit * 2)}` : null, includeCommandCommand: "bun scripts/cli.ts job list --include-command", statusCommand: "bun scripts/cli.ts job status --tail-bytes 12000", }, }; }