import { spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { repoRoot, rootPath } from "./config"; import { runCommandToFiles, tailFile } from "./command"; export type JobStatus = "queued" | "running" | "succeeded" | "failed"; export interface JobRecord { id: string; name: string; status: JobStatus; command: string[]; cwd: string; runner: "local" | "docker"; runnerPid?: number | null; runnerContainer?: string | null; createdAt: string; startedAt: string | null; finishedAt: string | null; exitCode: number | null; stdoutFile: string; stderrFile: string; note: string; } export interface JobProgressSummary { kind: "hwlab-v02-trigger" | "git-mirror" | "generic"; stage: string | null; stageStatus: string | null; sourceCommit: string | null; pipelineRun: string | null; pipelineCreated: boolean | null; elapsedSeconds: number | null; stageElapsedSeconds: number | null; lastEventAt: string | null; lastEventAgeSeconds: number | null; eventsObserved: number; slow: boolean; warnings: string[]; timings: Record; summary: string; nextCommand: string | null; } export interface StartJobOptions { runner?: "local" | "docker"; dockerImage?: string; } function jobsDir(): string { const dir = rootPath(".state", "jobs"); mkdirSync(dir, { recursive: true }); return dir; } function jobPath(id: string): string { return join(jobsDir(), `${id}.json`); } function writeJob(job: JobRecord): void { writeFileSync(jobPath(job.id), `${JSON.stringify(job, null, 2)}\n`, "utf8"); } export function readJob(id: string): JobRecord { const path = jobPath(id); if (!existsSync(path)) throw new Error(`job not found: ${id}`); return JSON.parse(readFileSync(path, "utf8")) as JobRecord; } export function listJobs(): JobRecord[] { return readdirSync(jobsDir()) .filter((name) => name.endsWith(".json")) .map((name) => JSON.parse(readFileSync(join(jobsDir(), name), "utf8")) as JobRecord) .sort((a, b) => b.createdAt.localeCompare(a.createdAt)); } export function startJob(name: string, command: string[], note: string, options: StartJobOptions = {}): JobRecord { const id = `${name}_${new Date().toISOString().replace(/[-:.TZ]/g, "")}_${Math.random().toString(16).slice(2, 8)}`; const stdoutFile = rootPath(".state", "jobs", `${id}.stdout.log`); const stderrFile = rootPath(".state", "jobs", `${id}.stderr.log`); const runner = options.runner ?? "local"; const job: JobRecord = { id, name, status: "queued", command, cwd: repoRoot, runner, runnerPid: null, runnerContainer: null, createdAt: new Date().toISOString(), startedAt: null, finishedAt: null, exitCode: null, stdoutFile, stderrFile, note, }; writeJob(job); if (runner === "docker") { const containerName = `unidesk-job-runner-${id}`.replace(/[^A-Za-z0-9_.-]/g, "-").slice(0, 120); job.runnerContainer = containerName; writeJob(job); const dockerArgs = [ "run", "-d", "--rm", "--name", containerName, "-v", "/var/run/docker.sock:/var/run/docker.sock", "-v", `${repoRoot}:${repoRoot}`, "-w", repoRoot, "--entrypoint", "bun", options.dockerImage ?? "unidesk-code-queue:latest", rootPath("scripts", "cli.ts"), "internal", "run-job", id, ]; const result = spawnSync("docker", dockerArgs, { cwd: repoRoot, encoding: "utf8" }); if (result.status !== 0) { job.status = "failed"; job.startedAt = new Date().toISOString(); job.finishedAt = new Date().toISOString(); job.exitCode = result.status ?? 127; writeFileSync(stderrFile, result.stderr || result.error?.message || "failed to start docker job runner\n", "utf8"); writeFileSync(stdoutFile, result.stdout || "", "utf8"); writeJob(job); } return job; } const child = spawn(process.execPath, [rootPath("scripts", "cli.ts"), "internal", "run-job", id], { cwd: repoRoot, detached: true, stdio: "ignore", env: process.env, }); job.runnerPid = child.pid ?? null; writeJob(job); child.unref(); return job; } export async function runJob(id: string): Promise { const job = readJob(id); job.status = "running"; job.runnerPid = process.pid; job.startedAt = new Date().toISOString(); writeJob(job); const exitCode = await runCommandToFiles(job.command, job.cwd, job.stdoutFile, job.stderrFile); job.exitCode = exitCode; job.status = exitCode === 0 ? "succeeded" : "failed"; job.finishedAt = new Date().toISOString(); writeJob(job); return job; } export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & { progress: JobProgressSummary; tailPolicy: { requestedTailBytes: number; stdoutBytes: number; stderrBytes: number; stdoutTruncated: boolean; stderrTruncated: boolean; fullLogPaths: { stdoutFile: string; stderrFile: string }; }; stdoutTail: string; stderrTail: string; } { const stdoutBytes = existsSync(job.stdoutFile) ? statSync(job.stdoutFile).size : 0; const stderrBytes = existsSync(job.stderrFile) ? statSync(job.stderrFile).size : 0; const progressTailBytes = Math.max(maxBytes, 96_000); const stdoutProgressTail = tailFile(job.stdoutFile, progressTailBytes); const stderrProgressTail = tailFile(job.stderrFile, progressTailBytes); return { ...job, progress: summarizeJobProgress(job, progressTailBytes, { stdoutTail: stdoutProgressTail, stderrTail: stderrProgressTail }), tailPolicy: { requestedTailBytes: maxBytes, stdoutBytes, stderrBytes, stdoutTruncated: stdoutBytes > maxBytes, stderrTruncated: stderrBytes > maxBytes, fullLogPaths: { stdoutFile: job.stdoutFile, stderrFile: job.stderrFile }, }, stdoutTail: tailTextByBytes(stdoutProgressTail, maxBytes), stderrTail: tailTextByBytes(stderrProgressTail, maxBytes), }; } function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary { const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current"; const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush"; if (!knownWorkflow && !gitMirrorWorkflow && tails === undefined) return genericJobProgress(job); const nowMs = Date.now(); const progressTailBytes = Math.max(4096, Math.floor(maxBytes)); const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes); const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes); if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs); const events = parseJsonLineEvents(stderrTail, "hwlab.v02.trigger.progress"); const lastEvent = events.at(-1) ?? {}; const stage = stringField(lastEvent.stage); const stageStatus = stringField(lastEvent.status); const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu); const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u); const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail) ? true : stage === "create-pipelinerun" && stageStatus === "failed" ? false : null; const lastEventAt = stringField(lastEvent.at); const kind = events.length > 0 || knownWorkflow ? "hwlab-v02-trigger" : "generic"; const elapsedSeconds = jobElapsedSeconds(job, nowMs); const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs); const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs); const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`); const warnings = jobProgressWarnings({ job, eventsObserved: events.length, elapsedSeconds, stage, stageStatus, stageElapsedSeconds, lastEventAgeSeconds, }); const slow = warnings.length > 0; const nextCommand = pipelineRun ? `bun scripts/cli.ts hwlab g14 control-plane status --lane v02` : job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null; const summary = kind === "hwlab-v02-trigger" ? [ job.status, stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown", sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null, pipelineRun ? `pipelineRun=${pipelineRun}` : null, pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null, elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null, lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null, slow ? "visibility-warning" : null, ].filter(Boolean).join(" ") : `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`; return { kind, stage, stageStatus, sourceCommit, pipelineRun, pipelineCreated, elapsedSeconds, stageElapsedSeconds, lastEventAt, lastEventAgeSeconds, eventsObserved: events.length, slow, warnings, timings, summary, nextCommand, }; } function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary { const action = job.name.endsWith("_flush") ? "flush" : job.name.endsWith("_sync") ? "sync" : "unknown"; const elapsedSeconds = jobElapsedSeconds(job, nowMs); const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`); const status = firstMatch(stdoutTail, new RegExp(`"event"\\s*:\\s*"git-mirror-${action}"[\\s\\S]{0,240}?"status"\\s*:\\s*"([^"]+)"`, "u")) ?? firstMatch(stdoutTail.replaceAll('\\"', '"').replaceAll("\\n", "\n"), new RegExp(`"event"\\s*:\\s*"git-mirror-${action}"[\\s\\S]{0,240}?"status"\\s*:\\s*"([^"]+)"`, "u")); const pendingFlush = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"pendingFlush"\s*:\s*(true|false)/u); const jobName = firstMatch(stdoutTail, /"jobName"\s*:\s*"([^"]+)"/u); const warnings = jobProgressWarnings({ job, eventsObserved: 0, elapsedSeconds, stage: action, stageStatus: job.status === "running" ? "running" : status, stageElapsedSeconds: job.status === "running" ? elapsedSeconds : null, lastEventAgeSeconds: null, }); const timingSummary = Object.entries(timings) .filter(([key]) => key.startsWith(`gitMirror.${action}.`) || key === "sshRuntimeMaxMs") .map(([key, value]) => `${key}=${value}ms`); return { kind: "git-mirror", stage: action, stageStatus: status ?? (job.status === "running" ? "running" : null), sourceCommit: null, pipelineRun: null, pipelineCreated: null, elapsedSeconds, stageElapsedSeconds: job.status === "running" ? elapsedSeconds : null, lastEventAt: null, lastEventAgeSeconds: null, eventsObserved: 0, slow: warnings.length > 0, warnings, timings, summary: [ job.status, `git-mirror-${action}${status ? `:${status}` : ""}`, jobName ? `job=${jobName}` : null, pendingFlush !== null ? `pendingFlush=${pendingFlush}` : null, elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, ...timingSummary, warnings.length > 0 ? "visibility-warning" : null, ].filter(Boolean).join(" "), nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : job.name.startsWith("agentrun_") ? "bun scripts/cli.ts agentrun v01 git-mirror status" : "bun scripts/cli.ts hwlab g14 git-mirror status", }; } function genericJobProgress(job: JobRecord): JobProgressSummary { const nowMs = Date.now(); const elapsedSeconds = jobElapsedSeconds(job, nowMs); const warnings = jobProgressWarnings({ job, eventsObserved: 0, elapsedSeconds, stage: null, stageStatus: null, stageElapsedSeconds: null, lastEventAgeSeconds: null, }); return { kind: "generic", stage: null, stageStatus: null, sourceCommit: null, pipelineRun: null, pipelineCreated: null, elapsedSeconds, stageElapsedSeconds: null, lastEventAt: null, lastEventAgeSeconds: null, eventsObserved: 0, slow: warnings.length > 0, warnings, timings: {}, summary: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`, nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null, }; } function jobElapsedSeconds(job: JobRecord, nowMs = Date.now()): number | null { const startMs = timestampMs(job.startedAt ?? job.createdAt); if (startMs === null) return null; const endMs = timestampMs(job.finishedAt) ?? nowMs; if (endMs < startMs) return null; return Math.round((endMs - startMs) / 1000); } function secondsSince(value: string, end: string | number = Date.now()): number | null { const startMs = timestampMs(value); const endMs = typeof end === "number" ? end : timestampMs(end); if (startMs === null || endMs === null || endMs < startMs) return null; return Math.round((endMs - startMs) / 1000); } function timestampMs(value: unknown): number | null { if (typeof value !== "string" || value.trim().length === 0) return null; const parsed = Date.parse(value); return Number.isFinite(parsed) ? parsed : null; } function currentStageElapsedSeconds(events: Record[], stage: string | null, stageStatus: string | null, job: JobRecord, nowMs = Date.now()): number | null { if (stage === null) return null; const stageEvents = events.filter((event) => stringField(event.stage) === stage); const startEvent = stageEvents.findLast((event) => stringField(event.status) === "started"); const startAt = stringField(startEvent?.at); if (startAt === null) return null; const endEvent = stageStatus === "started" ? null : stageEvents.findLast((event) => { const status = stringField(event.status); return status !== null && status !== "started"; }); const endAt = stringField(endEvent?.at) ?? job.finishedAt ?? (job.status === "running" ? new Date(nowMs).toISOString() : null); return endAt === null ? null : secondsSince(startAt, endAt); } function jobProgressWarnings(input: { job: JobRecord; eventsObserved: number; elapsedSeconds: number | null; stage: string | null; stageStatus: string | null; stageElapsedSeconds: number | null; lastEventAgeSeconds: number | null; }): string[] { if (input.job.status !== "running") return []; const warnings: string[] = []; if (input.eventsObserved === 0 && (input.elapsedSeconds ?? 0) >= 30) { warnings.push(`running for ${input.elapsedSeconds}s without progress events; inspect stdoutTail/stderrTail and fullLogPaths`); } if ((input.lastEventAgeSeconds ?? 0) >= 30) { warnings.push(`no progress event for ${input.lastEventAgeSeconds}s; current stage may be blocked or waiting on remote I/O`); } if (input.stage !== null && input.stageStatus !== null && input.stageStatus !== "succeeded" && input.stageStatus !== "failed" && (input.stageElapsedSeconds ?? 0) >= 30) { warnings.push(`stage ${input.stage}:${input.stageStatus} has been active for ${input.stageElapsedSeconds}s`); } return warnings; } function tailTextByBytes(text: string, maxBytes: number): string { const safeMaxBytes = Math.max(0, Math.floor(maxBytes)); if (safeMaxBytes === 0) return ""; const buffer = Buffer.from(text, "utf8"); if (buffer.length <= safeMaxBytes) return text; return buffer.subarray(buffer.length - safeMaxBytes).toString("utf8"); } function parseJsonLineEvents(text: string, eventName: string): Record[] { const events: Record[] = []; for (const line of text.split(/\r?\n/u)) { const trimmed = line.trim(); if (!trimmed.startsWith("{")) continue; try { const parsed = JSON.parse(trimmed) as unknown; if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) && (parsed as Record).event === eventName) { events.push(parsed as Record); } } catch { // Ignore non-JSON stderr lines; the raw tail remains available in stderrTail. } } return events; } function extractGitMirrorTimings(text: string): Record { const timings: Record = {}; for (const normalizedText of [text, text.replaceAll('\\"', '"').replaceAll("\\n", "\n")]) { extractGitMirrorTimingJsonLines(normalizedText, timings); extractGitMirrorTimingRegex(normalizedText, timings); extractSshRuntimeTimingRegex(normalizedText, timings); } return timings; } function extractGitMirrorTimingJsonLines(text: string, timings: Record): void { for (const line of text.split(/\r?\n/u)) { const trimmed = line.trim(); if (!trimmed.startsWith("{")) continue; try { const parsed = JSON.parse(trimmed) as unknown; if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) continue; const event = parsed as Record; const operation = event.event === "git-mirror-sync-timing" ? "sync" : event.event === "git-mirror-flush-timing" ? "flush" : null; if (operation === null) continue; const phase = stringField(event.phase); const durationMs = typeof event.durationMs === "number" && Number.isFinite(event.durationMs) ? event.durationMs : null; if (phase !== null && durationMs !== null) { timings[`gitMirror.${operation}.${phase}Ms`] = Math.round(durationMs); if (operation === "sync") timings[`gitMirror.${phase}Ms`] = Math.round(durationMs); } } catch { // Ignore non-JSON lines; raw logs remain available through job status tails. } } } function extractGitMirrorTimingRegex(text: string, timings: Record): void { const pattern = /"event"\s*:\s*"git-mirror-(sync|flush)-timing"[\s\S]{0,240}?"phase"\s*:\s*"([^"]+)"[\s\S]{0,240}?"durationMs"\s*:\s*(\d+)/gu; for (const match of text.matchAll(pattern)) { const operation = match[1]; const phase = match[2]; const durationMs = Number(match[3]); if (operation !== undefined && phase !== undefined && Number.isFinite(durationMs)) { timings[`gitMirror.${operation}.${phase}Ms`] = Math.round(durationMs); if (operation === "sync") timings[`gitMirror.${phase}Ms`] = Math.round(durationMs); } } } function extractSshRuntimeTimingRegex(text: string, timings: Record): void { const pattern = /"code"\s*:\s*"ssh-runtime-timing"[\s\S]{0,320}?"elapsedMs"\s*:\s*(\d+)/gu; const values = [...text.matchAll(pattern)] .map((match) => Number(match[1])) .filter((value) => Number.isFinite(value)); if (values.length > 0) timings.sshRuntimeMaxMs = Math.max(...values); } function stringField(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } function firstMatch(text: string, pattern: RegExp): string | null { const match = pattern.exec(text); return typeof match?.[1] === "string" && match[1].length > 0 ? match[1] : null; } export interface JobListOptions { limit?: number; includeCommand?: boolean; } export function listJobsSummary(options: JobListOptions = {}): unknown { const limit = Math.max(1, Math.floor(options.limit ?? 50)); const jobs = listJobs(); const returned = jobs.slice(0, limit).map((job) => ({ id: job.id, name: job.name, status: job.status, progress: summarizeJobProgress(job, 32_000), runner: job.runner, runnerPid: job.runnerPid ?? null, runnerContainer: job.runnerContainer ?? null, createdAt: job.createdAt, startedAt: job.startedAt, finishedAt: job.finishedAt, exitCode: job.exitCode, note: job.note, stdoutFile: job.stdoutFile, stderrFile: job.stderrFile, ...(options.includeCommand === true ? { command: job.command, cwd: job.cwd } : {}), })); return { jobs: returned, total: jobs.length, returned: returned.length, limit, truncated: jobs.length > returned.length, disclosure: { defaultLimit: 50, nextCommand: jobs.length > returned.length ? `bun scripts/cli.ts job list --limit ${Math.min(jobs.length, limit * 2)}` : null, includeCommandCommand: "bun scripts/cli.ts job list --include-command", statusCommand: "bun scripts/cli.ts job status --tail-bytes 12000", }, }; }