Lyon
1203 lines
52 KiB
TypeScript
1203 lines
52 KiB
TypeScript
// SPEC: PJ2026-01060509 出站诊断 draft-2026-06-26-p8-egress-job-friction.
|
|
// Async job state, summaries, and lifecycle drill-down rendering for UniDesk CLI.
|
|
import { spawn, spawnSync } from "node:child_process";
|
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { repoRoot, rootPath } from "./config";
|
|
import { runCommandToFiles, tailFile } from "./command";
|
|
import { hwlabDefaultRuntimeTarget } from "./hwlab-node-lanes";
|
|
import type { RenderedCliResult } from "./output";
|
|
|
|
export type JobStatus = "queued" | "running" | "succeeded" | "failed" | "canceled";
|
|
|
|
export interface JobRecord {
|
|
id: string;
|
|
name: string;
|
|
status: JobStatus;
|
|
command: string[];
|
|
cwd: string;
|
|
runner: "local" | "docker";
|
|
runnerPid?: number | null;
|
|
runnerContainer?: string | null;
|
|
createdAt: string;
|
|
startedAt: string | null;
|
|
finishedAt: string | null;
|
|
exitCode: number | null;
|
|
stdoutFile: string;
|
|
stderrFile: string;
|
|
note: string;
|
|
}
|
|
|
|
export interface JobProgressSummary {
|
|
kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "agentrun-yaml-lane-trigger" | "git-mirror" | "generic";
|
|
stage: string | null;
|
|
stageStatus: string | null;
|
|
sourceCommit: string | null;
|
|
pipelineRun: string | null;
|
|
pipelineCreated: boolean | null;
|
|
elapsedSeconds: number | null;
|
|
stageElapsedSeconds: number | null;
|
|
lastEventAt: string | null;
|
|
lastEventAgeSeconds: number | null;
|
|
eventsObserved: number;
|
|
slow: boolean;
|
|
warnings: string[];
|
|
diagnostics?: Record<string, unknown> | null;
|
|
timings: Record<string, number>;
|
|
summary: string;
|
|
nextCommand: string | null;
|
|
}
|
|
|
|
export interface StartJobOptions {
|
|
runner?: "local" | "docker";
|
|
dockerImage?: string;
|
|
}
|
|
|
|
function jobsDir(): string {
|
|
const dir = rootPath(".state", "jobs");
|
|
mkdirSync(dir, { recursive: true });
|
|
return dir;
|
|
}
|
|
|
|
function jobPath(id: string): string {
|
|
return join(jobsDir(), `${id}.json`);
|
|
}
|
|
|
|
function writeJob(job: JobRecord): void {
|
|
writeFileSync(jobPath(job.id), `${JSON.stringify(job, null, 2)}\n`, "utf8");
|
|
}
|
|
|
|
export function readJob(id: string): JobRecord {
|
|
const path = jobPath(id);
|
|
if (!existsSync(path)) throw new Error(`job not found: ${id}`);
|
|
return JSON.parse(readFileSync(path, "utf8")) as JobRecord;
|
|
}
|
|
|
|
export function cancelJob(id: string): unknown {
|
|
const job = readJob(id);
|
|
if (job.status !== "queued" && job.status !== "running") {
|
|
return { ok: true, action: "cancel-job", id, alreadyTerminal: true, job };
|
|
}
|
|
const actions: Array<Record<string, unknown>> = [];
|
|
if (job.runnerPid !== null && job.runnerPid !== undefined) {
|
|
try {
|
|
process.kill(-job.runnerPid, "SIGTERM");
|
|
actions.push({ signal: "SIGTERM", target: "process-group", pid: job.runnerPid, ok: true });
|
|
} catch (error) {
|
|
actions.push({ signal: "SIGTERM", target: "process-group", pid: job.runnerPid, ok: false, error: error instanceof Error ? error.message : String(error) });
|
|
try {
|
|
process.kill(job.runnerPid, "SIGTERM");
|
|
actions.push({ signal: "SIGTERM", target: "process", pid: job.runnerPid, ok: true });
|
|
} catch (innerError) {
|
|
actions.push({ signal: "SIGTERM", target: "process", pid: job.runnerPid, ok: false, error: innerError instanceof Error ? innerError.message : String(innerError) });
|
|
}
|
|
}
|
|
}
|
|
job.status = "canceled";
|
|
job.finishedAt = new Date().toISOString();
|
|
job.exitCode = 130;
|
|
writeFileSync(job.stderrFile, `${tailFile(job.stderrFile, 512_000)}${JSON.stringify({ event: "unidesk.job.cancel", at: job.finishedAt, jobId: id, actions })}\n`, "utf8");
|
|
writeJob(job);
|
|
return { ok: true, action: "cancel-job", id, actions, job };
|
|
}
|
|
|
|
export function listJobs(): JobRecord[] {
|
|
return readdirSync(jobsDir())
|
|
.filter((name) => name.endsWith(".json"))
|
|
.map((name) => JSON.parse(readFileSync(join(jobsDir(), name), "utf8")) as JobRecord)
|
|
.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
|
}
|
|
|
|
export function startJob(name: string, command: string[], note: string, options: StartJobOptions = {}): JobRecord {
|
|
const id = `${name}_${new Date().toISOString().replace(/[-:.TZ]/g, "")}_${Math.random().toString(16).slice(2, 8)}`;
|
|
const stdoutFile = rootPath(".state", "jobs", `${id}.stdout.log`);
|
|
const stderrFile = rootPath(".state", "jobs", `${id}.stderr.log`);
|
|
const runner = options.runner ?? "local";
|
|
const job: JobRecord = {
|
|
id,
|
|
name,
|
|
status: "queued",
|
|
command,
|
|
cwd: repoRoot,
|
|
runner,
|
|
runnerPid: null,
|
|
runnerContainer: null,
|
|
createdAt: new Date().toISOString(),
|
|
startedAt: null,
|
|
finishedAt: null,
|
|
exitCode: null,
|
|
stdoutFile,
|
|
stderrFile,
|
|
note,
|
|
};
|
|
writeJob(job);
|
|
if (runner === "docker") {
|
|
const containerName = `unidesk-job-runner-${id}`.replace(/[^A-Za-z0-9_.-]/g, "-").slice(0, 120);
|
|
job.runnerContainer = containerName;
|
|
writeJob(job);
|
|
const dockerArgs = [
|
|
"run",
|
|
"-d",
|
|
"--rm",
|
|
"--name",
|
|
containerName,
|
|
"-v",
|
|
"/var/run/docker.sock:/var/run/docker.sock",
|
|
"-v",
|
|
`${repoRoot}:${repoRoot}`,
|
|
"-w",
|
|
repoRoot,
|
|
"--entrypoint",
|
|
"bun",
|
|
options.dockerImage ?? "unidesk-code-queue:latest",
|
|
rootPath("scripts", "cli.ts"),
|
|
"internal",
|
|
"run-job",
|
|
id,
|
|
];
|
|
const result = spawnSync("docker", dockerArgs, { cwd: repoRoot, encoding: "utf8" });
|
|
if (result.status !== 0) {
|
|
job.status = "failed";
|
|
job.startedAt = new Date().toISOString();
|
|
job.finishedAt = new Date().toISOString();
|
|
job.exitCode = result.status ?? 127;
|
|
writeFileSync(stderrFile, result.stderr || result.error?.message || "failed to start docker job runner\n", "utf8");
|
|
writeFileSync(stdoutFile, result.stdout || "", "utf8");
|
|
writeJob(job);
|
|
}
|
|
return job;
|
|
}
|
|
const child = spawn(process.execPath, [rootPath("scripts", "cli.ts"), "internal", "run-job", id], {
|
|
cwd: repoRoot,
|
|
detached: true,
|
|
stdio: "ignore",
|
|
env: process.env,
|
|
});
|
|
job.runnerPid = child.pid ?? null;
|
|
writeJob(job);
|
|
child.unref();
|
|
return job;
|
|
}
|
|
|
|
export async function runJob(id: string): Promise<JobRecord> {
|
|
const job = readJob(id);
|
|
job.status = "running";
|
|
job.runnerPid = process.pid;
|
|
job.startedAt = new Date().toISOString();
|
|
writeJob(job);
|
|
const exitCode = await runCommandToFiles(job.command, job.cwd, job.stdoutFile, job.stderrFile);
|
|
job.exitCode = exitCode;
|
|
job.status = exitCode === 0 ? "succeeded" : "failed";
|
|
job.finishedAt = new Date().toISOString();
|
|
writeJob(job);
|
|
return job;
|
|
}
|
|
|
|
export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
|
|
progress: JobProgressSummary;
|
|
tailPolicy: {
|
|
requestedTailBytes: number;
|
|
stdoutBytes: number;
|
|
stderrBytes: number;
|
|
stdoutTruncated: boolean;
|
|
stderrTruncated: boolean;
|
|
fullLogPaths: { stdoutFile: string; stderrFile: string };
|
|
};
|
|
stdoutTail: string;
|
|
stderrTail: string;
|
|
} {
|
|
const stdoutBytes = existsSync(job.stdoutFile) ? statSync(job.stdoutFile).size : 0;
|
|
const stderrBytes = existsSync(job.stderrFile) ? statSync(job.stderrFile).size : 0;
|
|
const progressTailBytes = Math.max(maxBytes, 96_000);
|
|
const stdoutProgressTail = tailFile(job.stdoutFile, progressTailBytes);
|
|
const stderrProgressTail = tailFile(job.stderrFile, progressTailBytes);
|
|
const progress = summarizeJobProgress(job, progressTailBytes, { stdoutTail: stdoutProgressTail, stderrTail: stderrProgressTail });
|
|
const rawStdoutTail = tailTextByBytes(stdoutProgressTail, maxBytes);
|
|
const stdoutTail = compactJobStdoutTail(job, progress, rawStdoutTail, maxBytes);
|
|
return {
|
|
...job,
|
|
progress,
|
|
tailPolicy: {
|
|
requestedTailBytes: maxBytes,
|
|
stdoutBytes,
|
|
stderrBytes,
|
|
stdoutTruncated: stdoutBytes > maxBytes,
|
|
stderrTruncated: stderrBytes > maxBytes,
|
|
stdoutCompacted: stdoutTail !== rawStdoutTail,
|
|
fullLogPaths: { stdoutFile: job.stdoutFile, stderrFile: job.stderrFile },
|
|
},
|
|
stdoutTail,
|
|
stderrTail: tailTextByBytes(stderrProgressTail, maxBytes),
|
|
};
|
|
}
|
|
|
|
export function renderJobStatusSummary(job: ReturnType<typeof jobWithTail>): RenderedCliResult {
|
|
const progress = job.progress;
|
|
const warnings = Array.isArray(progress.warnings) ? progress.warnings : [];
|
|
const lines = [
|
|
jobStatusTable(
|
|
["JOB", "STATUS", "EXIT", "KIND", "STAGE", "STAGE_STATUS", "ELAPSED", "LAST_EVENT_AGE"],
|
|
[[job.id, job.status, job.exitCode ?? "-", progress.kind, progress.stage ?? "-", progress.stageStatus ?? "-", secondsText(progress.elapsedSeconds), secondsText(progress.lastEventAgeSeconds)]],
|
|
),
|
|
"",
|
|
jobStatusTable(
|
|
["SOURCE", "PIPELINE", "CREATED", "EVENTS", "SLOW", "SUMMARY"],
|
|
[[progress.sourceCommit ? String(progress.sourceCommit).slice(0, 12) : "-", progress.pipelineRun ?? "-", progress.pipelineCreated ?? "-", progress.eventsObserved, progress.slow, progress.summary]],
|
|
),
|
|
"",
|
|
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.slice(0, 6).map((item) => `- ${jobStatusCell(item, 220)}`)].join("\n"),
|
|
"",
|
|
jobStatusTable(
|
|
["TAIL", "BYTES", "TRUNCATED", "PATH"],
|
|
[
|
|
["stdout", job.tailPolicy.stdoutBytes, job.tailPolicy.stdoutTruncated, job.tailPolicy.fullLogPaths.stdoutFile],
|
|
["stderr", job.tailPolicy.stderrBytes, job.tailPolicy.stderrTruncated, job.tailPolicy.fullLogPaths.stderrFile],
|
|
],
|
|
),
|
|
"",
|
|
jobTailSection("STDOUT_TAIL", job.stdoutTail),
|
|
"",
|
|
jobTailSection("STDERR_TAIL", job.stderrTail),
|
|
"",
|
|
"NEXT",
|
|
` status: bun scripts/cli.ts job status ${job.id} --tail-bytes ${job.tailPolicy.requestedTailBytes}`,
|
|
progress.nextCommand ? ` progress: ${progress.nextCommand}` : null,
|
|
` full: bun scripts/cli.ts job status ${job.id} --tail-bytes ${job.tailPolicy.requestedTailBytes} --full`,
|
|
].filter((line): line is string => line !== null);
|
|
return {
|
|
ok: job.status !== "failed",
|
|
command: "job status",
|
|
contentType: "text/plain",
|
|
renderedText: `${lines.join("\n")}\n`,
|
|
};
|
|
}
|
|
|
|
export function renderJobLaunchSummary(command: string, result: unknown): RenderedCliResult | null {
|
|
if (typeof result !== "object" || result === null || !("job" in result)) return null;
|
|
const record = result as Record<string, unknown>;
|
|
const job = record.job as Partial<JobRecord> | undefined;
|
|
if (typeof job !== "object" || job === null || typeof job.id !== "string") return null;
|
|
const service = typeof record.service === "string" ? record.service : "-";
|
|
const runtimeEnv = typeof record.runtimeEnv === "object" && record.runtimeEnv !== null ? record.runtimeEnv as Record<string, unknown> : {};
|
|
const stdoutFile = typeof job.stdoutFile === "string" ? job.stdoutFile : "-";
|
|
const stderrFile = typeof job.stderrFile === "string" ? job.stderrFile : "-";
|
|
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
|
const fullCommand = command.includes("--full") || command.includes("--raw") ? null : `bun scripts/cli.ts ${command} --full`;
|
|
const lines = [
|
|
"ASYNC_JOB",
|
|
jobStatusTable(
|
|
["JOB", "NAME", "STATUS", "SERVICE", "RUNNER", "CREATED"],
|
|
[[job.id, job.name ?? "-", job.status ?? "-", service, job.runner ?? "-", job.createdAt ?? "-"]],
|
|
),
|
|
"",
|
|
jobStatusTable(
|
|
["STREAM", "PATH"],
|
|
[
|
|
["stdout", stdoutFile],
|
|
["stderr", stderrFile],
|
|
["runtimeEnv", typeof runtimeEnv.envFile === "string" ? runtimeEnv.envFile : "-"],
|
|
["logDir", typeof runtimeEnv.logDir === "string" ? runtimeEnv.logDir : "-"],
|
|
],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` status: ${statusCommand}`,
|
|
" latest: bun scripts/cli.ts job status latest --tail-bytes 12000",
|
|
" list: bun scripts/cli.ts job list --limit 20",
|
|
" logs: bun scripts/cli.ts server logs --tail-bytes 3000",
|
|
" server: bun scripts/cli.ts server status",
|
|
fullCommand === null ? null : ` full: ${fullCommand}`,
|
|
].filter((line): line is string => line !== null);
|
|
return {
|
|
ok: job.status !== "failed",
|
|
command,
|
|
contentType: "text/plain",
|
|
renderedText: `${lines.join("\n")}\n`,
|
|
};
|
|
}
|
|
|
|
function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary {
|
|
const nowMs = Date.now();
|
|
const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
|
|
const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor";
|
|
const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name);
|
|
const agentRunYamlLaneTriggerWorkflow = /^agentrun_v[0-9]{2}_trigger_current$/u.test(job.name);
|
|
const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync"
|
|
|| job.name === "hwlab_g14_git_mirror_flush"
|
|
|| job.name === "agentrun_v01_git_mirror_sync"
|
|
|| job.name === "agentrun_v01_git_mirror_flush"
|
|
|| /^hwlab_nodes_v[0-9]{2}_git-mirror_(sync|flush)$/u.test(job.name);
|
|
const ciInstallWorkflow = job.name === "ci_install";
|
|
if (ciInstallWorkflow) return summarizeCiInstallJobProgress(job, tails?.stderrTail, nowMs);
|
|
const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
|
|
const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes);
|
|
const agentRunYamlLaneProgressObserved = agentRunYamlLaneTriggerWorkflow || hasAgentRunYamlLaneProgressEvents(stderrTail);
|
|
if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !agentRunYamlLaneProgressObserved && !gitMirrorWorkflow) return genericJobProgress(job, stderrTail);
|
|
const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes);
|
|
if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs);
|
|
if (runtimeLaneTriggerWorkflow) return summarizeRuntimeLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs);
|
|
if (agentRunYamlLaneProgressObserved) return summarizeAgentRunYamlLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs);
|
|
const events = v02PrMonitorWorkflow
|
|
? [
|
|
...parseJsonLineEvents(stderrTail, "hwlab.v02.pr-monitor.progress"),
|
|
...parseJsonLineEvents(stderrTail, "hwlab.v02.trigger.progress"),
|
|
].sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")))
|
|
: parseJsonLineEvents(stderrTail, "hwlab.v02.trigger.progress");
|
|
const lastEvent = events.at(-1) ?? {};
|
|
const stage = stringField(lastEvent.stage);
|
|
const stageStatus = stringField(lastEvent.status);
|
|
const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu);
|
|
const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u);
|
|
const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail)
|
|
? true
|
|
: stage === "create-pipelinerun" && stageStatus === "failed"
|
|
? false
|
|
: null;
|
|
const lastEventAt = stringField(lastEvent.at);
|
|
const kind = events.length > 0 || knownWorkflow || v02PrMonitorWorkflow ? "hwlab-v02-trigger" : "generic";
|
|
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
|
|
const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs);
|
|
const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
|
|
const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`);
|
|
const warnings = jobProgressWarnings({
|
|
job,
|
|
eventsObserved: events.length,
|
|
elapsedSeconds,
|
|
stage,
|
|
stageStatus,
|
|
stageElapsedSeconds,
|
|
lastEventAgeSeconds,
|
|
});
|
|
const slow = warnings.length > 0;
|
|
const nextCommand = pipelineRun
|
|
? `bun scripts/cli.ts hwlab g14 control-plane status --lane v02`
|
|
: job.status === "running"
|
|
? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
|
|
: null;
|
|
const summary = kind === "hwlab-v02-trigger"
|
|
? [
|
|
job.status,
|
|
stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
|
|
sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
|
|
pipelineRun ? `pipelineRun=${pipelineRun}` : null,
|
|
pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null,
|
|
elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
|
|
stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
|
|
lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
|
|
slow ? "visibility-warning" : null,
|
|
].filter(Boolean).join(" ")
|
|
: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`;
|
|
return {
|
|
kind,
|
|
stage,
|
|
stageStatus,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
pipelineCreated,
|
|
elapsedSeconds,
|
|
stageElapsedSeconds,
|
|
lastEventAt,
|
|
lastEventAgeSeconds,
|
|
eventsObserved: events.length,
|
|
slow,
|
|
warnings,
|
|
timings,
|
|
summary,
|
|
nextCommand,
|
|
};
|
|
}
|
|
|
|
function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary {
|
|
const action = job.name.endsWith("_flush") ? "flush" : job.name.endsWith("_sync") ? "sync" : "unknown";
|
|
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
|
|
const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`);
|
|
const status =
|
|
firstMatch(stdoutTail, new RegExp(`"event"\\s*:\\s*"git-mirror-${action}"[\\s\\S]{0,240}?"status"\\s*:\\s*"([^"]+)"`, "u")) ??
|
|
firstMatch(stdoutTail.replaceAll('\\"', '"').replaceAll("\\n", "\n"), new RegExp(`"event"\\s*:\\s*"git-mirror-${action}"[\\s\\S]{0,240}?"status"\\s*:\\s*"([^"]+)"`, "u"));
|
|
const pendingFlush = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"pendingFlush"\s*:\s*(true|false)/u);
|
|
const partialSuccess = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"partialSuccess"\s*:\s*"([^"]+)"/u);
|
|
const jobName = firstMatch(stdoutTail, /"jobName"\s*:\s*"([^"]+)"/u);
|
|
const node = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"node"\s*:\s*"([^"]+)"/u);
|
|
const lane = firstMatch(stdoutTail.replaceAll('\\"', '"'), /"lane"\s*:\s*"(v[0-9]+)"/u) ?? firstMatch(job.name, /^hwlab_nodes_(v[0-9]+)_/u);
|
|
const warnings = jobProgressWarnings({
|
|
job,
|
|
eventsObserved: 0,
|
|
elapsedSeconds,
|
|
stage: action,
|
|
stageStatus: job.status === "running" ? "running" : status,
|
|
stageElapsedSeconds: job.status === "running" ? elapsedSeconds : null,
|
|
lastEventAgeSeconds: null,
|
|
});
|
|
const timingSummary = Object.entries(timings)
|
|
.filter(([key]) => key.startsWith(`gitMirror.${action}.`) || key === "sshRuntimeMaxMs")
|
|
.map(([key, value]) => `${key}=${value}ms`);
|
|
return {
|
|
kind: "git-mirror",
|
|
stage: action,
|
|
stageStatus: status ?? (job.status === "running" ? "running" : null),
|
|
sourceCommit: null,
|
|
pipelineRun: null,
|
|
pipelineCreated: null,
|
|
elapsedSeconds,
|
|
stageElapsedSeconds: job.status === "running" ? elapsedSeconds : null,
|
|
lastEventAt: null,
|
|
lastEventAgeSeconds: null,
|
|
eventsObserved: 0,
|
|
slow: warnings.length > 0,
|
|
warnings,
|
|
timings,
|
|
summary: [
|
|
job.status,
|
|
`git-mirror-${action}${status ? `:${status}` : ""}`,
|
|
partialSuccess ? `partialSuccess=${partialSuccess}` : null,
|
|
jobName ? `job=${jobName}` : null,
|
|
pendingFlush !== null ? `pendingFlush=${pendingFlush}` : null,
|
|
elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
|
|
...timingSummary,
|
|
warnings.length > 0 ? "visibility-warning" : null,
|
|
].filter(Boolean).join(" "),
|
|
nextCommand: job.status === "running"
|
|
? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
|
|
: job.name.startsWith("agentrun_")
|
|
? "bun scripts/cli.ts agentrun git-mirror status"
|
|
: job.name.startsWith("hwlab_nodes_") && node !== null && lane !== null
|
|
? partialSuccess === "push-succeeded-fetch-failed"
|
|
? `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${node} --lane ${lane} --confirm --wait`
|
|
: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`
|
|
: "bun scripts/cli.ts hwlab g14 git-mirror status",
|
|
};
|
|
}
|
|
|
|
function summarizeCiInstallJobProgress(job: JobRecord, stderrTailOverride?: string, nowMs = Date.now()): JobProgressSummary {
|
|
const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000);
|
|
const events = parseJsonLineEvents(stderrTail, "ci.install.progress")
|
|
.sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")));
|
|
const lastEvent = events.at(-1) ?? {};
|
|
const stage = stringField(lastEvent.stage);
|
|
const stageStatus = stringField(lastEvent.status);
|
|
const lastEventAt = stringField(lastEvent.at);
|
|
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
|
|
const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs);
|
|
const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
|
|
const warnings = jobProgressWarnings({
|
|
job,
|
|
eventsObserved: events.length,
|
|
elapsedSeconds,
|
|
stage,
|
|
stageStatus,
|
|
stageElapsedSeconds,
|
|
lastEventAgeSeconds,
|
|
});
|
|
return {
|
|
kind: "generic",
|
|
stage,
|
|
stageStatus,
|
|
sourceCommit: null,
|
|
pipelineRun: null,
|
|
pipelineCreated: null,
|
|
elapsedSeconds,
|
|
stageElapsedSeconds,
|
|
lastEventAt,
|
|
lastEventAgeSeconds,
|
|
eventsObserved: events.length,
|
|
slow: warnings.length > 0,
|
|
warnings,
|
|
timings: {},
|
|
summary: [
|
|
job.status,
|
|
stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
|
|
typeof lastEvent.providerId === "string" ? `provider=${lastEvent.providerId}` : null,
|
|
typeof lastEvent.manifest === "string" ? `manifest=${lastEvent.manifest}` : null,
|
|
elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
|
|
stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
|
|
lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
|
|
warnings.length > 0 ? "visibility-warning" : null,
|
|
].filter(Boolean).join(" "),
|
|
nextCommand: job.status === "running"
|
|
? `bun scripts/cli.ts ci install-status ${job.id}`
|
|
: "bun scripts/cli.ts ci status",
|
|
};
|
|
}
|
|
|
|
function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary {
|
|
const events = parseJsonLineEvents(stderrTail, "hwlab.runtime-lane.trigger.progress")
|
|
.sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")));
|
|
const lastEvent = events.at(-1) ?? {};
|
|
const stage = stringField(lastEvent.stage);
|
|
const stageStatus = stringField(lastEvent.status);
|
|
const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu);
|
|
const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u);
|
|
const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail)
|
|
? true
|
|
: stage === "create-pipelinerun" && stageStatus === "failed"
|
|
? false
|
|
: null;
|
|
const lastEventAt = stringField(lastEvent.at);
|
|
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
|
|
const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs);
|
|
const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
|
|
const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`);
|
|
const warnings = jobProgressWarnings({
|
|
job,
|
|
eventsObserved: events.length,
|
|
elapsedSeconds,
|
|
stage,
|
|
stageStatus,
|
|
stageElapsedSeconds,
|
|
lastEventAgeSeconds,
|
|
});
|
|
const nextStatusCommand = hwlabRuntimeLaneStatusNextCommand(pipelineRun, lastEvent);
|
|
if (nextStatusCommand.warning !== null) warnings.push(nextStatusCommand.warning);
|
|
const tcpPoolDiagnostics = job.status === "succeeded" ? null : sshTcpPoolDiagnosticsFromJobText(`${stderrTail}\n${stdoutTail}`);
|
|
const pipelineFailureDiagnostics = runtimeLanePipelineFailureDiagnostics(stdoutTail);
|
|
const slow = warnings.length > 0;
|
|
const diagnostics = pipelineFailureDiagnostics !== null
|
|
? { pipelineFailure: pipelineFailureDiagnostics, sshTcpPool: tcpPoolDiagnostics }
|
|
: tcpPoolDiagnostics;
|
|
return {
|
|
kind: "hwlab-runtime-lane-trigger",
|
|
stage,
|
|
stageStatus,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
pipelineCreated,
|
|
elapsedSeconds,
|
|
stageElapsedSeconds,
|
|
lastEventAt,
|
|
lastEventAgeSeconds,
|
|
eventsObserved: events.length,
|
|
slow,
|
|
warnings,
|
|
diagnostics,
|
|
timings,
|
|
summary: [
|
|
job.status,
|
|
stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
|
|
sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
|
|
pipelineRun ? `pipelineRun=${pipelineRun}` : null,
|
|
pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null,
|
|
pipelineFailureDiagnostics !== null ? runtimeLanePipelineFailureSummaryText(pipelineFailureDiagnostics) : null,
|
|
elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
|
|
stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
|
|
lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
|
|
tcpPoolDiagnostics !== null ? `sshPool=${String(tcpPoolDiagnostics.failureKind ?? "transient")}` : null,
|
|
slow ? "visibility-warning" : null,
|
|
].filter(Boolean).join(" "),
|
|
nextCommand: pipelineRun
|
|
? nextStatusCommand.command
|
|
: job.status === "running"
|
|
? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
|
|
: null,
|
|
};
|
|
}
|
|
|
|
const agentRunYamlLaneProgressEvents = [
|
|
{ event: "agentrun.yaml-lane.source-bootstrap.progress", stage: "source-bootstrap" },
|
|
{ event: "agentrun.yaml-lane.image-build.progress", stage: "image-build" },
|
|
{ event: "agentrun.yaml-lane.gitops-publish.progress", stage: "gitops-publish" },
|
|
{ event: "agentrun.yaml-lane.git-mirror.progress", stage: "git-mirror" },
|
|
] as const;
|
|
|
|
function hasAgentRunYamlLaneProgressEvents(text: string): boolean {
|
|
return parseAgentRunYamlLaneProgressEvents(text).length > 0;
|
|
}
|
|
|
|
function summarizeAgentRunYamlLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary {
|
|
const events = parseAgentRunYamlLaneProgressEvents(stderrTail);
|
|
const lastEvent = events.at(-1) ?? {};
|
|
const stage = stringField(lastEvent.stage);
|
|
const stageStatus = agentRunYamlLaneStageStatus(lastEvent);
|
|
const sourceCommit = lastStringField(events, "sourceCommit") ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu);
|
|
const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u);
|
|
const pipelineCreatedValue = firstMatch(stdoutTail, /"created"\s*:\s*\{[\s\S]{0,400}?"created"\s*:\s*(true|false)/u);
|
|
const pipelineCreated = pipelineCreatedValue === "true" ? true : pipelineCreatedValue === "false" ? false : null;
|
|
const lastEventAt = stringField(lastEvent.at);
|
|
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
|
|
const eventElapsedMs = numberField(lastEvent.elapsedMs);
|
|
const stageElapsedSeconds = eventElapsedMs === null ? null : Math.round(eventElapsedMs / 1000);
|
|
const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
|
|
const timings = agentRunYamlLaneTimings(events);
|
|
const warnings = jobProgressWarnings({
|
|
job,
|
|
eventsObserved: events.length,
|
|
elapsedSeconds,
|
|
stage,
|
|
stageStatus,
|
|
stageElapsedSeconds,
|
|
lastEventAgeSeconds,
|
|
});
|
|
const slow = warnings.length > 0;
|
|
const eventNode = stringField(lastEvent.node);
|
|
const eventLane = stringField(lastEvent.lane) ?? firstMatch(job.name, /^agentrun_(v[0-9]{2})_trigger_current$/u);
|
|
const nextCommand = pipelineRun
|
|
? [
|
|
"bun scripts/cli.ts agentrun control-plane status",
|
|
eventNode ? `--node ${eventNode}` : null,
|
|
eventLane ? `--lane ${eventLane}` : null,
|
|
`--pipeline-run ${pipelineRun}`,
|
|
].filter(Boolean).join(" ")
|
|
: job.status === "running"
|
|
? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
|
|
: null;
|
|
return {
|
|
kind: "agentrun-yaml-lane-trigger",
|
|
stage,
|
|
stageStatus,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
pipelineCreated,
|
|
elapsedSeconds,
|
|
stageElapsedSeconds,
|
|
lastEventAt,
|
|
lastEventAgeSeconds,
|
|
eventsObserved: events.length,
|
|
slow,
|
|
warnings,
|
|
diagnostics: {
|
|
stages: agentRunYamlLaneStageDiagnostics(events),
|
|
},
|
|
timings,
|
|
summary: [
|
|
job.status,
|
|
stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
|
|
sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
|
|
pipelineRun ? `pipelineRun=${pipelineRun}` : null,
|
|
pipelineCreated === true ? "created" : pipelineCreated === false ? "create-reused" : null,
|
|
elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
|
|
stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
|
|
lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
|
|
`events=${events.length}`,
|
|
slow ? "visibility-warning" : null,
|
|
].filter(Boolean).join(" "),
|
|
nextCommand,
|
|
};
|
|
}
|
|
|
|
function parseAgentRunYamlLaneProgressEvents(text: string): Record<string, unknown>[] {
|
|
const eventToStage = new Map(agentRunYamlLaneProgressEvents.map((item) => [item.event, item.stage]));
|
|
const events: Record<string, unknown>[] = [];
|
|
const lines = text.split(/\r?\n/u);
|
|
for (let index = 0; index < lines.length; index += 1) {
|
|
const line = lines[index]?.trim() ?? "";
|
|
if (line.length === 0) continue;
|
|
if (line.startsWith("{")) {
|
|
try {
|
|
const parsed = JSON.parse(line) as unknown;
|
|
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
|
|
const record = parsed as Record<string, unknown>;
|
|
const event = typeof record.event === "string" ? record.event : null;
|
|
const stage = event === null ? null : eventToStage.get(event) ?? null;
|
|
if (event !== null && stage !== null) events.push({ ...record, stage, __order: index });
|
|
}
|
|
} catch {
|
|
// Ignore non-JSON progress noise; raw tails remain visible.
|
|
}
|
|
continue;
|
|
}
|
|
const match = line.match(/^status\s+(agentrun\.yaml-lane\.[a-z-]+\.progress)\s+([a-z-]+)(?:\s+(.*))?$/u);
|
|
if (!match) continue;
|
|
const event = match[1]!;
|
|
const stage = eventToStage.get(event);
|
|
if (stage === undefined) continue;
|
|
const payload: Record<string, unknown> = { event, stage, status: match[2]!, __order: index };
|
|
for (const token of (match[3] ?? "").split(/\s+/u).filter(Boolean)) {
|
|
const tokenMatch = token.match(/^([A-Za-z][A-Za-z0-9_]*)=(.+)$/u);
|
|
if (!tokenMatch) continue;
|
|
const key = tokenMatch[1]!;
|
|
const value = tokenMatch[2]!;
|
|
payload[key] = /^-?\d+$/u.test(value) ? Number(value) : value;
|
|
}
|
|
events.push(payload);
|
|
}
|
|
return events.sort((left, right) => Number(left.__order ?? 0) - Number(right.__order ?? 0));
|
|
}
|
|
|
|
function agentRunYamlLaneStageStatus(event: Record<string, unknown>): string | null {
|
|
const explicitStatus = stringField(event.status);
|
|
if (explicitStatus !== null) return explicitStatus;
|
|
if (event.succeeded === true) return "succeeded";
|
|
if (event.failed === true) return "failed";
|
|
if (event.stage !== undefined) return "running";
|
|
return null;
|
|
}
|
|
|
|
function agentRunYamlLaneTimings(events: Record<string, unknown>[]): Record<string, number> {
|
|
const timings: Record<string, number> = {};
|
|
for (const event of events) {
|
|
const stage = stringField(event.stage);
|
|
const elapsedMs = numberField(event.elapsedMs);
|
|
if (stage !== null && elapsedMs !== null) timings[`agentrunYamlLane.${stage}.elapsedMs`] = Math.round(elapsedMs);
|
|
}
|
|
return timings;
|
|
}
|
|
|
|
function agentRunYamlLaneStageDiagnostics(events: Record<string, unknown>[]): Array<Record<string, unknown>> {
|
|
return agentRunYamlLaneProgressEvents
|
|
.map(({ stage }) => {
|
|
const stageEvents = events.filter((event) => stringField(event.stage) === stage);
|
|
const lastEvent = stageEvents.at(-1) ?? {};
|
|
return {
|
|
stage,
|
|
eventsObserved: stageEvents.length,
|
|
status: agentRunYamlLaneStageStatus(lastEvent),
|
|
jobId: stringField(lastEvent.jobId),
|
|
jobName: stringField(lastEvent.jobName),
|
|
polls: numberField(lastEvent.polls),
|
|
elapsedMs: numberField(lastEvent.elapsedMs),
|
|
};
|
|
})
|
|
.filter((item) => item.eventsObserved > 0);
|
|
}
|
|
|
|
function hwlabRuntimeLaneStatusNextCommand(pipelineRun: string | null, lastEvent: Record<string, unknown>): { command: string | null; warning: string | null } {
|
|
if (pipelineRun === null) return { command: null, warning: null };
|
|
const eventNode = stringField(lastEvent.node);
|
|
const eventLane = stringField(lastEvent.lane);
|
|
const defaults = hwlabDefaultRuntimeTarget();
|
|
const node = eventNode ?? defaults.node;
|
|
const lane = eventLane ?? defaults.lane;
|
|
const missing = [
|
|
eventNode === null ? "node" : null,
|
|
eventLane === null ? "lane" : null,
|
|
].filter((value): value is string => value !== null);
|
|
return {
|
|
command: `bun scripts/cli.ts hwlab nodes control-plane status --node ${node} --lane ${lane} --pipeline-run ${pipelineRun}`,
|
|
warning: missing.length === 0
|
|
? null
|
|
: `runtime lane progress event missing ${missing.join("/")} for nextCommand; used YAML default ${defaults.node}/${defaults.lane} from config/hwlab-node-lanes.yaml`,
|
|
};
|
|
}
|
|
|
|
function sshTcpPoolDiagnosticsFromJobText(text: string): Record<string, unknown> | null {
|
|
const hint = lastJsonLinePayload(text, "UNIDESK_SSH_TCP_POOL_HINT");
|
|
if (hint !== null) return hint;
|
|
const error = lastJsonLinePayload(text, "UNIDESK_SSH_ERROR");
|
|
if (error !== null && typeof error.failureKind === "string" && error.failureKind.startsWith("provider-data-")) {
|
|
return {
|
|
code: "ssh-tcp-pool-transient",
|
|
failureKind: error.failureKind,
|
|
providerId: error.providerId ?? null,
|
|
dataChannelId: error.dataChannelId ?? null,
|
|
dataPool: error.dataPool ?? null,
|
|
diagnostics: { fullHealth: "bun scripts/cli.ts debug health" },
|
|
};
|
|
}
|
|
if (/ssh tcp data channel closed/iu.test(text)) {
|
|
return { code: "ssh-tcp-pool-transient", failureKind: "provider-data-channel-closed", diagnostics: { fullHealth: "bun scripts/cli.ts debug health" } };
|
|
}
|
|
if (/requested ssh tcp data channel is not ready|ssh tcp data channel is not available/iu.test(text)) {
|
|
return { code: "ssh-tcp-pool-transient", failureKind: "provider-data-channel-missing", diagnostics: { fullHealth: "bun scripts/cli.ts debug health" } };
|
|
}
|
|
if (/provider ssh tcp data pool has no idle channel/iu.test(text)) {
|
|
return { code: "ssh-tcp-pool-transient", failureKind: "provider-data-pool-exhausted", diagnostics: { fullHealth: "bun scripts/cli.ts debug health" } };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function lastJsonLinePayload(text: string, prefix: string): Record<string, unknown> | null {
|
|
const lines = text.split(/\r?\n/u);
|
|
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
const line = lines[index]?.trim() ?? "";
|
|
if (!line.startsWith(`${prefix} `)) continue;
|
|
try {
|
|
const parsed = JSON.parse(line.slice(prefix.length + 1)) as unknown;
|
|
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) return parsed as Record<string, unknown>;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function runtimeLanePipelineFailureDiagnostics(stdoutTail: string): Record<string, unknown> | null {
|
|
const parsed = parseJsonObjectFromText(stdoutTail);
|
|
const data = recordField(parsed.data);
|
|
const directSummary = recordField(data.pipelineFailureSummary);
|
|
const waitSummary = recordField(recordField(data.pipelineWait).failureSummary);
|
|
const statusSummary = recordField(recordField(recordField(data.pipelineRun).diagnostics).failureSummary);
|
|
const failureSummary = Object.keys(directSummary).length > 0
|
|
? directSummary
|
|
: Object.keys(waitSummary).length > 0
|
|
? waitSummary
|
|
: statusSummary;
|
|
const firstFailure = recordField(failureSummary.firstFailure);
|
|
if (Object.keys(failureSummary).length > 0) {
|
|
return {
|
|
degradedReason: data.degradedReason ?? null,
|
|
failedTaskRunCount: failureSummary.failedTaskRunCount ?? null,
|
|
failedStepCount: failureSummary.failedStepCount ?? null,
|
|
stepPublishFailureCount: failureSummary.stepPublishFailureCount ?? null,
|
|
firstFailure: compactRuntimeLaneFailure(firstFailure),
|
|
nextAction: failureSummary.nextAction ?? null,
|
|
};
|
|
}
|
|
const taskRun = firstMatch(stdoutTail, /"taskRun"\s*:\s*"([^"]+)"/u);
|
|
const pod = firstMatch(stdoutTail, /"pod"\s*:\s*"([^"]+)"/u);
|
|
const step = firstMatch(stdoutTail, /"step"\s*:\s*"([^"]+)"/u);
|
|
const container = firstMatch(stdoutTail, /"container"\s*:\s*"([^"]+)"/u);
|
|
if (taskRun === null && pod === null && step === null && container === null) return null;
|
|
return {
|
|
degradedReason: firstMatch(stdoutTail, /"degradedReason"\s*:\s*"([^"]+)"/u),
|
|
failedTaskRunCount: null,
|
|
failedStepCount: null,
|
|
stepPublishFailureCount: container === "step-publish" || step === "publish" || step === "step-publish" ? 1 : 0,
|
|
firstFailure: { taskRun, pod, step, container, logCommand: firstMatch(stdoutTail, /"logCommand"\s*:\s*"([^"]+)"/u) },
|
|
nextAction: null,
|
|
};
|
|
}
|
|
|
|
function runtimeLanePipelineFailureSummaryText(diagnostics: Record<string, unknown>): string | null {
|
|
const firstFailure = recordField(diagnostics.firstFailure);
|
|
const taskRun = stringField(firstFailure.taskRun);
|
|
const step = stringField(firstFailure.step);
|
|
const container = stringField(firstFailure.container);
|
|
const stepPublishCount = numberField(diagnostics.stepPublishFailureCount);
|
|
const failedTaskRunCount = numberField(diagnostics.failedTaskRunCount);
|
|
const parts = [
|
|
failedTaskRunCount !== null ? `failedTaskRuns=${failedTaskRunCount}` : null,
|
|
taskRun !== null ? `taskRun=${taskRun}` : null,
|
|
step !== null ? `step=${step}` : null,
|
|
container !== null ? `container=${container}` : null,
|
|
stepPublishCount !== null && stepPublishCount > 0 ? "step-publish" : null,
|
|
];
|
|
const text = parts.filter(Boolean).join(" ");
|
|
return text.length > 0 ? text : null;
|
|
}
|
|
|
|
function compactRuntimeLaneFailure(value: Record<string, unknown>): Record<string, unknown> | null {
|
|
if (Object.keys(value).length === 0) return null;
|
|
return {
|
|
taskRun: value.taskRun ?? null,
|
|
pipelineTask: value.pipelineTask ?? null,
|
|
taskRef: value.taskRef ?? null,
|
|
pod: value.pod ?? null,
|
|
step: value.step ?? null,
|
|
container: value.container ?? null,
|
|
terminationReason: value.terminationReason ?? null,
|
|
exitCode: value.exitCode ?? null,
|
|
logCommand: value.logCommand ?? null,
|
|
taskRunCommand: value.taskRunCommand ?? null,
|
|
podDescribeCommand: value.podDescribeCommand ?? null,
|
|
};
|
|
}
|
|
|
|
function parseJsonObjectFromText(text: string): Record<string, unknown> {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return {};
|
|
try {
|
|
return recordField(JSON.parse(trimmed) as unknown);
|
|
} catch {
|
|
const start = trimmed.indexOf("{");
|
|
const end = trimmed.lastIndexOf("}");
|
|
if (start >= 0 && end > start) {
|
|
try {
|
|
return recordField(JSON.parse(trimmed.slice(start, end + 1)) as unknown);
|
|
} catch {}
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
|
|
function recordField(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary {
|
|
const nowMs = Date.now();
|
|
const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000);
|
|
const downloadProgressEvents = parseJsonLineEvents(stderrTail, "unidesk.ssh.download.progress");
|
|
const downloadRetryEvents = [
|
|
...parseJsonLineEvents(stderrTail, "unidesk.ssh.download.empty-read-retry"),
|
|
...parseJsonLineEvents(stderrTail, "unidesk.ssh.download.short-read-retry"),
|
|
];
|
|
const downloadEvents = [...downloadProgressEvents, ...downloadRetryEvents].sort((left, right) => Number(left.blockIndex ?? -1) - Number(right.blockIndex ?? -1));
|
|
const lastDownload = downloadEvents.at(-1) ?? null;
|
|
const lastDownloadAt = stringField(lastDownload?.at);
|
|
const lastEventAgeSeconds = lastDownloadAt === null ? null : secondsSince(lastDownloadAt, job.finishedAt ?? nowMs);
|
|
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
|
|
const warnings = jobProgressWarnings({
|
|
job,
|
|
eventsObserved: downloadEvents.length,
|
|
elapsedSeconds,
|
|
stage: lastDownload === null ? null : "ssh-download",
|
|
stageStatus: lastDownload === null ? null : "running",
|
|
stageElapsedSeconds: null,
|
|
lastEventAgeSeconds,
|
|
});
|
|
const downloadSummary = lastDownload === null
|
|
? null
|
|
: [
|
|
` ssh-download ${Number(lastDownload.actualBytes ?? lastDownload.bytes ?? 0)}/${Number(lastDownload.expectedBytes ?? lastDownload.totalBytes ?? 0)} bytes`,
|
|
`chunks=${Number(lastDownload.chunks ?? 0)}`,
|
|
`remaining=${Number(lastDownload.remainingBytes ?? 0)}`,
|
|
`throughput=${Number(lastDownload.throughputBytesPerSecond ?? 0)}Bps`,
|
|
`elapsedMs=${Number(lastDownload.elapsedMs ?? 0)}`,
|
|
downloadRetryEvents.length === 0 ? null : `retries=${downloadRetryEvents.length} lastRetry=${String(lastDownload.reason ?? "unknown")}`,
|
|
].filter(Boolean).join(" ");
|
|
return {
|
|
kind: "generic",
|
|
stage: lastDownload === null ? null : "ssh-download",
|
|
stageStatus: lastDownload === null ? null : "running",
|
|
sourceCommit: null,
|
|
pipelineRun: null,
|
|
pipelineCreated: null,
|
|
elapsedSeconds,
|
|
stageElapsedSeconds: null,
|
|
lastEventAt: lastDownloadAt,
|
|
lastEventAgeSeconds,
|
|
eventsObserved: downloadEvents.length,
|
|
slow: warnings.length > 0,
|
|
warnings,
|
|
timings: {},
|
|
summary: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}${downloadSummary ?? ""}`,
|
|
nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null,
|
|
};
|
|
}
|
|
|
|
function jobElapsedSeconds(job: JobRecord, nowMs = Date.now()): number | null {
|
|
const startMs = timestampMs(job.startedAt ?? job.createdAt);
|
|
if (startMs === null) return null;
|
|
const endMs = timestampMs(job.finishedAt) ?? nowMs;
|
|
if (endMs < startMs) return null;
|
|
return Math.round((endMs - startMs) / 1000);
|
|
}
|
|
|
|
function secondsSince(value: string, end: string | number = Date.now()): number | null {
|
|
const startMs = timestampMs(value);
|
|
const endMs = typeof end === "number" ? end : timestampMs(end);
|
|
if (startMs === null || endMs === null || endMs < startMs) return null;
|
|
return Math.round((endMs - startMs) / 1000);
|
|
}
|
|
|
|
function timestampMs(value: unknown): number | null {
|
|
if (typeof value !== "string" || value.trim().length === 0) return null;
|
|
const parsed = Date.parse(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function currentStageElapsedSeconds(events: Record<string, unknown>[], stage: string | null, stageStatus: string | null, job: JobRecord, nowMs = Date.now()): number | null {
|
|
if (stage === null) return null;
|
|
const stageEvents = events.filter((event) => stringField(event.stage) === stage);
|
|
const startEvent = stageEvents.findLast((event) => stringField(event.status) === "started");
|
|
const startAt = stringField(startEvent?.at);
|
|
if (startAt === null) return null;
|
|
const endEvent = stageStatus === "started"
|
|
? null
|
|
: stageEvents.findLast((event) => {
|
|
const status = stringField(event.status);
|
|
return status !== null && status !== "started";
|
|
});
|
|
const endAt = stringField(endEvent?.at) ?? job.finishedAt ?? (job.status === "running" ? new Date(nowMs).toISOString() : null);
|
|
return endAt === null ? null : secondsSince(startAt, endAt);
|
|
}
|
|
|
|
function jobProgressWarnings(input: {
|
|
job: JobRecord;
|
|
eventsObserved: number;
|
|
elapsedSeconds: number | null;
|
|
stage: string | null;
|
|
stageStatus: string | null;
|
|
stageElapsedSeconds: number | null;
|
|
lastEventAgeSeconds: number | null;
|
|
}): string[] {
|
|
if (input.job.status !== "running") return [];
|
|
if (input.stage === "idle" && input.stageStatus === "waiting") return [];
|
|
const warnings: string[] = [];
|
|
if (input.eventsObserved === 0 && (input.elapsedSeconds ?? 0) >= 30) {
|
|
warnings.push(`running for ${input.elapsedSeconds}s without progress events; inspect stdoutTail/stderrTail and fullLogPaths`);
|
|
}
|
|
if ((input.lastEventAgeSeconds ?? 0) >= 30) {
|
|
warnings.push(`no progress event for ${input.lastEventAgeSeconds}s; current stage may be blocked or waiting on remote I/O`);
|
|
}
|
|
if (input.stage !== null && input.stageStatus !== null && input.stageStatus !== "succeeded" && input.stageStatus !== "failed" && (input.stageElapsedSeconds ?? 0) >= 30) {
|
|
warnings.push(`stage ${input.stage}:${input.stageStatus} has been active for ${input.stageElapsedSeconds}s`);
|
|
}
|
|
return warnings;
|
|
}
|
|
|
|
function tailTextByBytes(text: string, maxBytes: number): string {
|
|
const safeMaxBytes = Math.max(0, Math.floor(maxBytes));
|
|
if (safeMaxBytes === 0) return "";
|
|
const buffer = Buffer.from(text, "utf8");
|
|
if (buffer.length <= safeMaxBytes) return text;
|
|
return buffer.subarray(buffer.length - safeMaxBytes).toString("utf8");
|
|
}
|
|
|
|
function secondsText(value: number | null): string {
|
|
return value === null ? "-" : `${value}s`;
|
|
}
|
|
|
|
function jobStatusCell(value: unknown, maxLength = 96): string {
|
|
if (value === null || value === undefined) return "-";
|
|
const text = typeof value === "string" ? value : String(value);
|
|
const compact = text.replace(/\s+/gu, " ").trim();
|
|
if (compact.length === 0) return "-";
|
|
if (compact.length <= maxLength) return compact;
|
|
return `${compact.slice(0, Math.max(1, maxLength - 1))}...`;
|
|
}
|
|
|
|
function jobStatusTable(headers: string[], rows: unknown[][]): string {
|
|
const stringRows = rows.map((row) => row.map((value) => jobStatusCell(value)));
|
|
const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0)));
|
|
const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd();
|
|
return [renderRow(headers), ...stringRows.map(renderRow)].join("\n");
|
|
}
|
|
|
|
function jobTailSection(title: string, text: string): string {
|
|
const lines = text.trim().split(/\r?\n/u).filter((line) => line.trim().length > 0).slice(-6).map((line) => ` ${jobStatusCell(line, 220)}`);
|
|
return lines.length === 0 ? `${title}\n-` : [title, ...lines].join("\n");
|
|
}
|
|
|
|
function compactJobStdoutTail(job: JobRecord, progress: JobProgressSummary, rawTail: string, requestedTailBytes: number): string {
|
|
if (progress.kind !== "hwlab-runtime-lane-trigger") return rawTail;
|
|
if (Buffer.byteLength(rawTail, "utf8") <= 4096 && !rawTail.includes("\"expected\"") && !rawTail.includes("stdoutTail")) return rawTail;
|
|
const interestingLines = rawTail.split(/\r?\n/u)
|
|
.map((line) => line.trimEnd())
|
|
.filter((line) => !/"(stdoutTail|stderrTail|expected|result)"\s*:/u.test(line))
|
|
.filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|pipelineFailureSummary|failureSummary|firstFailure|failedTaskRun|stepFailure|taskRun|pod|step|container|terminationReason|exitCode|logCommand|taskRunCommand|podDescribeCommand|step-publish|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line))
|
|
.slice(-80);
|
|
const header = [
|
|
`[stdout compacted: ${progress.kind}; requestedTailBytes=${requestedTailBytes}]`,
|
|
`full stdout: ${job.stdoutFile}`,
|
|
].join("\n");
|
|
if (interestingLines.length === 0) return `${header}\n${tailTextByBytes(rawTail, Math.min(requestedTailBytes, 3000))}`;
|
|
return `${header}\n${interestingLines.join("\n")}\n`;
|
|
}
|
|
|
|
function parseJsonLineEvents(text: string, eventName: string): Record<string, unknown>[] {
|
|
const events: Record<string, unknown>[] = [];
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed.startsWith("{")) continue;
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) && (parsed as Record<string, unknown>).event === eventName) {
|
|
events.push(parsed as Record<string, unknown>);
|
|
}
|
|
} catch {
|
|
// Ignore non-JSON stderr lines; the raw tail remains available in stderrTail.
|
|
}
|
|
}
|
|
return events;
|
|
}
|
|
|
|
function extractGitMirrorTimings(text: string): Record<string, number> {
|
|
const timings: Record<string, number> = {};
|
|
for (const normalizedText of [text, text.replaceAll('\\"', '"').replaceAll("\\n", "\n")]) {
|
|
extractGitMirrorTimingJsonLines(normalizedText, timings);
|
|
extractGitMirrorTimingRegex(normalizedText, timings);
|
|
extractSshRuntimeTimingRegex(normalizedText, timings);
|
|
}
|
|
return timings;
|
|
}
|
|
|
|
function extractGitMirrorTimingJsonLines(text: string, timings: Record<string, number>): void {
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed.startsWith("{")) continue;
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) continue;
|
|
const event = parsed as Record<string, unknown>;
|
|
const operation = event.event === "git-mirror-sync-timing"
|
|
? "sync"
|
|
: event.event === "git-mirror-flush-timing"
|
|
? "flush"
|
|
: null;
|
|
if (operation === null) continue;
|
|
const phase = stringField(event.phase);
|
|
const durationMs = typeof event.durationMs === "number" && Number.isFinite(event.durationMs) ? event.durationMs : null;
|
|
if (phase !== null && durationMs !== null) {
|
|
timings[`gitMirror.${operation}.${phase}Ms`] = Math.round(durationMs);
|
|
if (operation === "sync") timings[`gitMirror.${phase}Ms`] = Math.round(durationMs);
|
|
}
|
|
} catch {
|
|
// Ignore non-JSON lines; raw logs remain available through job status tails.
|
|
}
|
|
}
|
|
}
|
|
|
|
function extractGitMirrorTimingRegex(text: string, timings: Record<string, number>): void {
|
|
const pattern = /"event"\s*:\s*"git-mirror-(sync|flush)-timing"[\s\S]{0,240}?"phase"\s*:\s*"([^"]+)"[\s\S]{0,240}?"durationMs"\s*:\s*(\d+)/gu;
|
|
for (const match of text.matchAll(pattern)) {
|
|
const operation = match[1];
|
|
const phase = match[2];
|
|
const durationMs = Number(match[3]);
|
|
if (operation !== undefined && phase !== undefined && Number.isFinite(durationMs)) {
|
|
timings[`gitMirror.${operation}.${phase}Ms`] = Math.round(durationMs);
|
|
if (operation === "sync") timings[`gitMirror.${phase}Ms`] = Math.round(durationMs);
|
|
}
|
|
}
|
|
}
|
|
|
|
function extractSshRuntimeTimingRegex(text: string, timings: Record<string, number>): void {
|
|
const pattern = /"code"\s*:\s*"ssh-runtime-timing"[\s\S]{0,320}?"elapsedMs"\s*:\s*(\d+)/gu;
|
|
const values = [...text.matchAll(pattern)]
|
|
.map((match) => Number(match[1]))
|
|
.filter((value) => Number.isFinite(value));
|
|
if (values.length > 0) timings.sshRuntimeMaxMs = Math.max(...values);
|
|
}
|
|
|
|
function stringField(value: unknown): string | null {
|
|
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
}
|
|
|
|
function numberField(value: unknown): number | null {
|
|
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
if (typeof value === "string" && value.trim().length > 0) {
|
|
const parsed = Number(value);
|
|
if (Number.isFinite(parsed)) return parsed;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function lastStringField(events: Record<string, unknown>[], field: string): string | null {
|
|
const event = events.findLast((item) => stringField(item[field]) !== null);
|
|
return stringField(event?.[field]);
|
|
}
|
|
|
|
function firstMatch(text: string, pattern: RegExp): string | null {
|
|
const match = pattern.exec(text);
|
|
return typeof match?.[1] === "string" && match[1].length > 0 ? match[1] : null;
|
|
}
|
|
|
|
export interface JobListOptions {
|
|
limit?: number;
|
|
includeCommand?: boolean;
|
|
}
|
|
|
|
export function listJobsSummary(options: JobListOptions = {}): unknown {
|
|
const limit = Math.max(1, Math.floor(options.limit ?? 50));
|
|
const jobs = listJobs();
|
|
const returned = jobs.slice(0, limit).map((job) => ({
|
|
id: job.id,
|
|
name: job.name,
|
|
status: job.status,
|
|
progress: summarizeJobProgress(job, 32_000),
|
|
runner: job.runner,
|
|
runnerPid: job.runnerPid ?? null,
|
|
runnerContainer: job.runnerContainer ?? null,
|
|
createdAt: job.createdAt,
|
|
startedAt: job.startedAt,
|
|
finishedAt: job.finishedAt,
|
|
exitCode: job.exitCode,
|
|
note: job.note,
|
|
stdoutFile: job.stdoutFile,
|
|
stderrFile: job.stderrFile,
|
|
...(options.includeCommand === true ? { command: job.command, cwd: job.cwd } : {}),
|
|
}));
|
|
return {
|
|
jobs: returned,
|
|
total: jobs.length,
|
|
returned: returned.length,
|
|
limit,
|
|
truncated: jobs.length > returned.length,
|
|
disclosure: {
|
|
defaultLimit: 50,
|
|
nextCommand: jobs.length > returned.length ? `bun scripts/cli.ts job list --limit ${Math.min(jobs.length, limit * 2)}` : null,
|
|
includeCommandCommand: "bun scripts/cli.ts job list --include-command",
|
|
statusCommand: "bun scripts/cli.ts job status <jobId> --tail-bytes 12000",
|
|
},
|
|
};
|
|
}
|