fix: improve AgentRun status visibility
This commit is contained in:
Codex
+135
-2
@@ -26,7 +26,7 @@ export interface JobRecord {
|
||||
}
|
||||
|
||||
export interface JobProgressSummary {
|
||||
kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "git-mirror" | "generic";
|
||||
kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "agentrun-yaml-lane-trigger" | "git-mirror" | "generic";
|
||||
stage: string | null;
|
||||
stageStatus: string | null;
|
||||
sourceCommit: string | null;
|
||||
@@ -229,15 +229,17 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou
|
||||
const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
|
||||
const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor";
|
||||
const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name);
|
||||
const agentRunYamlLaneTriggerWorkflow = /^agentrun_v[0-9]{2}_trigger_current$/u.test(job.name);
|
||||
const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush";
|
||||
const ciInstallWorkflow = job.name === "ci_install";
|
||||
if (ciInstallWorkflow) return summarizeCiInstallJobProgress(job, tails?.stderrTail, nowMs);
|
||||
if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
|
||||
if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !agentRunYamlLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
|
||||
const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
|
||||
const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes);
|
||||
const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes);
|
||||
if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs);
|
||||
if (runtimeLaneTriggerWorkflow) return summarizeRuntimeLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs);
|
||||
if (agentRunYamlLaneTriggerWorkflow) return summarizeAgentRunYamlLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs);
|
||||
const events = v02PrMonitorWorkflow
|
||||
? [
|
||||
...parseJsonLineEvents(stderrTail, "hwlab.v02.pr-monitor.progress"),
|
||||
@@ -479,6 +481,123 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri
|
||||
};
|
||||
}
|
||||
|
||||
const agentRunYamlLaneProgressEvents = [
|
||||
{ event: "agentrun.yaml-lane.source-bootstrap.progress", stage: "source-bootstrap" },
|
||||
{ event: "agentrun.yaml-lane.image-build.progress", stage: "image-build" },
|
||||
{ event: "agentrun.yaml-lane.gitops-publish.progress", stage: "gitops-publish" },
|
||||
{ event: "agentrun.yaml-lane.git-mirror.progress", stage: "git-mirror" },
|
||||
] as const;
|
||||
|
||||
function summarizeAgentRunYamlLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary {
|
||||
const events = agentRunYamlLaneProgressEvents
|
||||
.flatMap(({ event, stage }) => parseJsonLineEvents(stderrTail, event).map((item) => ({ ...item, stage })))
|
||||
.sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")));
|
||||
const lastEvent = events.at(-1) ?? {};
|
||||
const stage = stringField(lastEvent.stage);
|
||||
const stageStatus = agentRunYamlLaneStageStatus(lastEvent);
|
||||
const sourceCommit = lastStringField(events, "sourceCommit") ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu);
|
||||
const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u);
|
||||
const pipelineCreatedValue = firstMatch(stdoutTail, /"created"\s*:\s*\{[\s\S]{0,400}?"created"\s*:\s*(true|false)/u);
|
||||
const pipelineCreated = pipelineCreatedValue === "true" ? true : pipelineCreatedValue === "false" ? false : null;
|
||||
const lastEventAt = stringField(lastEvent.at);
|
||||
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
|
||||
const eventElapsedMs = numberField(lastEvent.elapsedMs);
|
||||
const stageElapsedSeconds = eventElapsedMs === null ? null : Math.round(eventElapsedMs / 1000);
|
||||
const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
|
||||
const timings = agentRunYamlLaneTimings(events);
|
||||
const warnings = jobProgressWarnings({
|
||||
job,
|
||||
eventsObserved: events.length,
|
||||
elapsedSeconds,
|
||||
stage,
|
||||
stageStatus,
|
||||
stageElapsedSeconds,
|
||||
lastEventAgeSeconds,
|
||||
});
|
||||
const slow = warnings.length > 0;
|
||||
const eventNode = stringField(lastEvent.node);
|
||||
const eventLane = stringField(lastEvent.lane) ?? firstMatch(job.name, /^agentrun_(v[0-9]{2})_trigger_current$/u);
|
||||
const nextCommand = pipelineRun
|
||||
? [
|
||||
"bun scripts/cli.ts agentrun control-plane status",
|
||||
eventNode ? `--node ${eventNode}` : null,
|
||||
eventLane ? `--lane ${eventLane}` : null,
|
||||
`--pipeline-run ${pipelineRun}`,
|
||||
].filter(Boolean).join(" ")
|
||||
: job.status === "running"
|
||||
? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
|
||||
: null;
|
||||
return {
|
||||
kind: "agentrun-yaml-lane-trigger",
|
||||
stage,
|
||||
stageStatus,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
pipelineCreated,
|
||||
elapsedSeconds,
|
||||
stageElapsedSeconds,
|
||||
lastEventAt,
|
||||
lastEventAgeSeconds,
|
||||
eventsObserved: events.length,
|
||||
slow,
|
||||
warnings,
|
||||
diagnostics: {
|
||||
stages: agentRunYamlLaneStageDiagnostics(events),
|
||||
},
|
||||
timings,
|
||||
summary: [
|
||||
job.status,
|
||||
stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
|
||||
sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
|
||||
pipelineRun ? `pipelineRun=${pipelineRun}` : null,
|
||||
pipelineCreated === true ? "created" : pipelineCreated === false ? "create-reused" : null,
|
||||
elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
|
||||
stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
|
||||
lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
|
||||
`events=${events.length}`,
|
||||
slow ? "visibility-warning" : null,
|
||||
].filter(Boolean).join(" "),
|
||||
nextCommand,
|
||||
};
|
||||
}
|
||||
|
||||
function agentRunYamlLaneStageStatus(event: Record<string, unknown>): string | null {
|
||||
const explicitStatus = stringField(event.status);
|
||||
if (explicitStatus !== null) return explicitStatus;
|
||||
if (event.succeeded === true) return "succeeded";
|
||||
if (event.failed === true) return "failed";
|
||||
if (event.stage !== undefined) return "running";
|
||||
return null;
|
||||
}
|
||||
|
||||
function agentRunYamlLaneTimings(events: Record<string, unknown>[]): Record<string, number> {
|
||||
const timings: Record<string, number> = {};
|
||||
for (const event of events) {
|
||||
const stage = stringField(event.stage);
|
||||
const elapsedMs = numberField(event.elapsedMs);
|
||||
if (stage !== null && elapsedMs !== null) timings[`agentrunYamlLane.${stage}.elapsedMs`] = Math.round(elapsedMs);
|
||||
}
|
||||
return timings;
|
||||
}
|
||||
|
||||
function agentRunYamlLaneStageDiagnostics(events: Record<string, unknown>[]): Array<Record<string, unknown>> {
|
||||
return agentRunYamlLaneProgressEvents
|
||||
.map(({ stage }) => {
|
||||
const stageEvents = events.filter((event) => stringField(event.stage) === stage);
|
||||
const lastEvent = stageEvents.at(-1) ?? {};
|
||||
return {
|
||||
stage,
|
||||
eventsObserved: stageEvents.length,
|
||||
status: agentRunYamlLaneStageStatus(lastEvent),
|
||||
jobId: stringField(lastEvent.jobId),
|
||||
jobName: stringField(lastEvent.jobName),
|
||||
polls: numberField(lastEvent.polls),
|
||||
elapsedMs: numberField(lastEvent.elapsedMs),
|
||||
};
|
||||
})
|
||||
.filter((item) => item.eventsObserved > 0);
|
||||
}
|
||||
|
||||
function hwlabRuntimeLaneStatusNextCommand(pipelineRun: string | null, lastEvent: Record<string, unknown>): { command: string | null; warning: string | null } {
|
||||
if (pipelineRun === null) return { command: null, warning: null };
|
||||
const eventNode = stringField(lastEvent.node);
|
||||
@@ -738,6 +857,20 @@ function stringField(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function numberField(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isFinite(value)) return value;
|
||||
if (typeof value === "string" && value.trim().length > 0) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isFinite(parsed)) return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function lastStringField(events: Record<string, unknown>[], field: string): string | null {
|
||||
const event = events.findLast((item) => stringField(item[field]) !== null);
|
||||
return stringField(event?.[field]);
|
||||
}
|
||||
|
||||
function firstMatch(text: string, pattern: RegExp): string | null {
|
||||
const match = pattern.exec(text);
|
||||
return typeof match?.[1] === "string" && match[1].length > 0 ? match[1] : null;
|
||||
|
||||
Reference in New Issue
Block a user