Merge pull request #1506 from pikasTech/codex/agentrun-reconcile-timeline-1501

Add branch follower reconcile timeline visibility

scripts/src/cicd-branch-follower.ts

@@ -31,6 +31,7 @@ import { invalidRuntimeReuseConfig, missingRuntimeReuseConfig, parseRuntimeReuse
 import { prioritizedTaskRunItems } from "./cicd-taskruns";
 import { runBranchFollowerTaskRunDrillDown } from "./cicd-taskrun-drilldown";
 import { runBranchFollowerJobDrillDown, runBranchFollowerRuntimeDrillDown } from "./cicd-job-runtime-drilldown";
+import { attachReconcileTimeline, compactReconcileTimeline, finishReconcileStep, finishReconcileTimeline, startReconcileStep, startReconcileTimeline } from "./cicd-reconcile-timeline";
 import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types";
 import {
   arrayField,
@@ -686,17 +687,27 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions)
     };
   }
   const selected = selectFollowers(registry, options, { includeDisabled: false });
+  const reconcileTimeline = startReconcileTimeline({ controller: options.inCluster, dryRun: options.dryRun, confirm: options.confirm, wait: options.wait, followerIds: selected.map((follower) => follower.id) });
+  const stateReadStep = startReconcileStep(reconcileTimeline, "*", "state-read");
   const previous = readK8sState(registry, options);
+  finishReconcileStep(stateReadStep, { status: previous.ok ? "ok" : "degraded", object: registry.controller.stateConfigMapName, reason: previous.errors.join("; ") });
   const results: FollowerState[] = [];
   const stateWriteWarnings: string[] = [];
   const stateWrites: Record<string, unknown>[] = [];
   for (const follower of selected) {
     const oldState = previous.stateByFollower[follower.id] ?? {};
+    const statusReadStep = startReconcileStep(reconcileTimeline, follower.id, "status-read");
     const live = await readAdapterStatus(registry, follower, options);
+    finishReconcileStep(statusReadStep, { status: live.ok ? "ok" : "failed", observedSha: live.observedSha, targetSha: live.targetSha, phase: live.phase, pipelineRun: live.pipelineRun, message: live.message });
+    const decideStep = startReconcileStep(reconcileTimeline, follower.id, "decide");
     const state = await decideAndMaybeTrigger(registry, follower, oldState, live, options);
+    finishReconcileStep(decideStep, { status: state.phase === "Failed" || state.phase === "Blocked" ? "blocked" : "ok", observedSha: state.source.observedSha, targetSha: state.target.targetSha, phase: state.phase, pipelineRun: state.pipelineRun, message: state.decision });
     if (!options.dryRun || options.recordState) {
+      state.command = attachReconcileTimeline(state.command, reconcileTimeline, follower.id);
+      const pendingWriteStep = startReconcileStep(reconcileTimeline, follower.id, "state-write");
       const write = writeFollowerState(registry, state, options);
-      const writeSummary = stateWriteSummary(follower.id, write);
+      finishReconcileStep(pendingWriteStep, { status: write.exitCode === 0 ? "ok" : "failed", object: registry.controller.stateConfigMapName, exitCode: write.exitCode, reason: write.stderr || write.stdout });
+      const writeSummary = stateWriteSummary(follower.id, write, pendingWriteStep.step.elapsedMs);
       stateWrites.push(writeSummary);
       if (write.exitCode !== 0) {
         const warning = `state write failed for ${follower.id}: ${tailText(write.stderr || write.stdout, 300)}`;
@@ -706,6 +717,7 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions)
     }
     results.push(state);
   }
+  finishReconcileTimeline(reconcileTimeline);
   return {
     ok: results.every((item) => item.phase !== "Failed" && item.phase !== "Blocked"),
     action: "run-once",
@@ -714,6 +726,7 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions)
     wait: options.wait,
     controller: options.inCluster,
     registry: registrySummary(registry),
+    reconcileTimeline: compactReconcileTimeline(reconcileTimeline),
     followers: results,
     stateWrites,
     warnings: stateWriteWarnings,
@@ -1937,6 +1950,11 @@ function mergeFollowerStatus(
   const lastTriggeredSha = live?.lastTriggeredSha ?? stringOrNull(stored.lastTriggeredSha);
   const lastSucceededSha = live?.lastSucceededSha ?? stringOrNull(stored.lastSucceededSha);
   const timings = live === null ? storedFollowerTimingsForStatus(follower, asOptionalRecord(stored.timings), phase, observedSha) : buildFollowerTimings(follower, live, undefined, asOptionalRecord(stored.timings), phase);
+  const reconcileTimeline = compactReconcileTimeline(asOptionalRecord(stored.command)?.reconcileTimeline, follower.id) ?? {
+    bounded: true,
+    missingReason: "stored state lacks reconcileTimeline; old data cannot be reconstructed",
+    steps: [],
+  };
   const summary: Record<string, unknown> = {
     ok: live === null ? true : live.ok,
     id: follower.id,
@@ -1963,6 +1981,7 @@ function mergeFollowerStatus(
     live: liveRequested,
     message: live?.message ?? stringOrNull(stored.decision) ?? "no controller state yet",
     timings: detailed ? timings : compactListTimings(timings),
+    reconcileTimeline: detailed ? reconcileTimeline : null,
     drilldown: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --live`,
   };
   if (!detailed) return summary;
@@ -2063,7 +2082,7 @@ function kubeConfigMapFollowerState(registry: BranchFollowerRegistry, options: P
   };
 }
 
-function stateWriteSummary(followerId: string, result: CommandResult): Record<string, unknown> {
+function stateWriteSummary(followerId: string, result: CommandResult, elapsedMs?: number): Record<string, unknown> {
   const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null;
   return {
     follower: followerId,
@@ -2073,6 +2092,7 @@ function stateWriteSummary(followerId: string, result: CommandResult): Record<st
     beforeResourceVersion: stringOrNull(parsed?.beforeResourceVersion),
     afterResourceVersion: stringOrNull(parsed?.afterResourceVersion),
     preservedTiming: parsed?.preservedTiming === true,
+    elapsedMs: typeof elapsedMs === "number" && Number.isFinite(elapsedMs) ? elapsedMs : null,
     message: result.exitCode === 0 ? "state patch command completed" : redactText(tailText(result.stderr || result.stdout, 500)),
     parsedDownstreamCliOutput: false,
   };
@@ -2222,6 +2242,7 @@ function compactStateCommand(command: Record<string, unknown> | undefined): Reco
     exitCode: numberOrNull(command.exitCode),
     timedOut: command.timedOut === true,
     statusAuthority: stringOrNull(command.statusAuthority),
+    reconcileTimeline: compactReconcileTimeline(command.reconcileTimeline),
     parsedDownstreamCliOutput: false,
   };
 }

scripts/src/cicd-reconcile-timeline.ts

@@ -0,0 +1,171 @@
+// SPEC: PJ2026-01060703 CI/CD branch follower reconcile timeline visibility.
+// Responsibility: bounded controller-loop timing summaries for branch-follower state/status output.
+
+export type ReconcileTimelineStep = {
+  follower: string;
+  step: string;
+  status: string;
+  startedAt: string;
+  finishedAt?: string;
+  elapsedMs?: number;
+  observedSha?: string;
+  targetSha?: string;
+  phase?: string;
+  pipelineRun?: string;
+  object?: string;
+  message?: string;
+  reason?: string;
+  exitCode?: number;
+};
+
+export type ReconcileTimeline = {
+  startedAt: string;
+  finishedAt?: string;
+  elapsedMs?: number;
+  controller: boolean;
+  dryRun: boolean;
+  confirm: boolean;
+  wait: boolean;
+  followerCount: number;
+  followers: string[];
+  bounded: true;
+  steps: ReconcileTimelineStep[];
+};
+
+export type ReconcileStepMarker = {
+  readonly step: ReconcileTimelineStep;
+  readonly startedMs: number;
+};
+
+export function startReconcileTimeline(input: { controller: boolean; dryRun: boolean; confirm: boolean; wait: boolean; followerIds: string[] }): ReconcileTimeline {
+  return {
+    startedAt: new Date().toISOString(),
+    controller: input.controller,
+    dryRun: input.dryRun,
+    confirm: input.confirm,
+    wait: input.wait,
+    followerCount: input.followerIds.length,
+    followers: input.followerIds.slice(0, 8),
+    bounded: true,
+    steps: [],
+  };
+}
+
+export function finishReconcileTimeline(timeline: ReconcileTimeline): ReconcileTimeline {
+  const finishedMs = Date.now();
+  timeline.finishedAt = new Date(finishedMs).toISOString();
+  timeline.elapsedMs = elapsedMs(timeline.startedAt, finishedMs);
+  timeline.steps = compactSteps(timeline.steps, null, 32);
+  return timeline;
+}
+
+export function startReconcileStep(timeline: ReconcileTimeline, follower: string, step: string): ReconcileStepMarker {
+  const startedMs = Date.now();
+  const record: ReconcileTimelineStep = {
+    follower: safeText(follower, 80) ?? "-",
+    step: safeText(step, 80) ?? "-",
+    status: "running",
+    startedAt: new Date(startedMs).toISOString(),
+  };
+  timeline.steps.push(record);
+  return { step: record, startedMs };
+}
+
+export function finishReconcileStep(marker: ReconcileStepMarker, fields: Record<string, unknown> = {}): ReconcileTimelineStep {
+  const finishedMs = Date.now();
+  marker.step.finishedAt = new Date(finishedMs).toISOString();
+  marker.step.elapsedMs = Math.max(0, finishedMs - marker.startedMs);
+  marker.step.status = safeText(fields.status, 80) ?? (fields.ok === false ? "failed" : "ok");
+  setText(marker.step, "observedSha", fields.observedSha, 80);
+  setText(marker.step, "targetSha", fields.targetSha, 80);
+  setText(marker.step, "phase", fields.phase, 80);
+  setText(marker.step, "pipelineRun", fields.pipelineRun, 120);
+  setText(marker.step, "object", fields.object, 120);
+  setText(marker.step, "message", fields.message, 180);
+  setText(marker.step, "reason", fields.reason, 180);
+  const exitCode = numberOrNull(fields.exitCode);
+  if (exitCode !== null) marker.step.exitCode = exitCode;
+  return marker.step;
+}
+
+export function attachReconcileTimeline(command: Record<string, unknown> | undefined, timeline: ReconcileTimeline, followerId: string): Record<string, unknown> | undefined {
+  const compact = compactReconcileTimeline(timeline, followerId);
+  if (compact === null) return command;
+  return { ...(command ?? {}), reconcileTimeline: compact };
+}
+
+export function compactReconcileTimeline(value: unknown, followerId?: string | null): Record<string, unknown> | null {
+  const source = asRecord(value);
+  if (source === null) return null;
+  const steps = compactSteps(arrayRecords(source.steps), followerId ?? null, followerId === undefined || followerId === null ? 32 : 16);
+  return {
+    startedAt: safeText(source.startedAt, 80),
+    finishedAt: safeText(source.finishedAt, 80),
+    elapsedMs: numberOrNull(source.elapsedMs),
+    controller: source.controller === true,
+    dryRun: source.dryRun === true,
+    confirm: source.confirm === true,
+    wait: source.wait === true,
+    followerCount: numberOrNull(source.followerCount),
+    followers: Array.isArray(source.followers) ? source.followers.map((item) => safeText(item, 80)).filter((item): item is string => item !== null).slice(0, 8) : [],
+    bounded: true,
+    omittedStepCount: Math.max(0, arrayRecords(source.steps).length - steps.length),
+    steps,
+  };
+}
+
+function compactSteps(values: Record<string, unknown>[], followerId: string | null, maxSteps: number): ReconcileTimelineStep[] {
+  const filtered = values
+    .filter((item) => followerId === null || item.follower === "*" || item.follower === followerId)
+    .slice(-maxSteps);
+  return filtered.map((item) => {
+    const step: ReconcileTimelineStep = {
+      follower: safeText(item.follower, 80) ?? "-",
+      step: safeText(item.step, 80) ?? "-",
+      status: safeText(item.status, 80) ?? "-",
+      startedAt: safeText(item.startedAt, 80) ?? "-",
+    };
+    setText(step, "finishedAt", item.finishedAt, 80);
+    const elapsed = numberOrNull(item.elapsedMs);
+    if (elapsed !== null) step.elapsedMs = elapsed;
+    setText(step, "observedSha", item.observedSha, 80);
+    setText(step, "targetSha", item.targetSha, 80);
+    setText(step, "phase", item.phase, 80);
+    setText(step, "pipelineRun", item.pipelineRun, 120);
+    setText(step, "object", item.object, 120);
+    setText(step, "message", item.message, 180);
+    setText(step, "reason", item.reason, 180);
+    const exitCode = numberOrNull(item.exitCode);
+    if (exitCode !== null) step.exitCode = exitCode;
+    return step;
+  });
+}
+
+function setText(target: ReconcileTimelineStep, key: keyof ReconcileTimelineStep, value: unknown, maxLength: number): void {
+  const text = safeText(value, maxLength);
+  if (text !== null) (target as Record<string, unknown>)[key] = text;
+}
+
+function elapsedMs(startedAt: string, finishedMs: number): number | undefined {
+  const startedMs = Date.parse(startedAt);
+  if (!Number.isFinite(startedMs)) return undefined;
+  return Math.max(0, finishedMs - startedMs);
+}
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
+}
+
+function arrayRecords(value: unknown): Record<string, unknown>[] {
+  return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
+}
+
+function numberOrNull(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function safeText(value: unknown, maxLength: number): string | null {
+  if (typeof value !== "string" || value.length === 0) return null;
+  const text = value.replace(/\s+/gu, " ");
+  return text.length <= maxLength ? text : `${text.slice(0, Math.max(0, maxLength - 3))}...`;
+}

scripts/src/cicd-render.ts

@@ -110,6 +110,7 @@ function renderStatusHuman(payload: Record<string, unknown>, _options: ParsedOpt
   const next = asOptionalRecord(payload.next);
   const errors = Array.isArray(payload.errors) ? payload.errors : [];
   const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48);
+  const reconcileRows = followers.flatMap(reconcileRowsForFollower).slice(0, 48);
   return [
     `CI/CD BRANCH-FOLLOWER STATUS (${payload.ok === false ? "degraded" : "ok"})`,
     "",
@@ -120,6 +121,7 @@ function renderStatusHuman(payload: Record<string, unknown>, _options: ParsedOpt
     "",
     table(["FOLLOWER", "PHASE", "ADAPTER", "OBSERVED", "TARGET", "TRIGGERED", "SUCCEEDED", "IN_FLIGHT", "BUDGET", "MESSAGE"], rows),
     timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`,
+    reconcileRows.length === 0 ? "" : `\nRECONCILE TIMELINE\n${table(["FOLLOWER", "STEP", "STATUS", "SECONDS", "STARTED", "OBJECT"], reconcileRows)}`,
     errors.length === 0 ? "" : `\nERRORS\n${errors.map((item) => `- ${item}`).join("\n")}`,
     "",
     "NEXT",
@@ -147,6 +149,7 @@ function renderRunOnceHuman(payload: Record<string, unknown>): string {
   });
   const next = asOptionalRecord(payload.next);
   const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48);
+  const reconcileRows = reconcileRowsFromRunOnce(payload, followers).slice(0, 48);
   const writeRows = stateWrites.map((item) => [
     item.follower,
     item.ok === true ? "ok" : "failed",
@@ -161,6 +164,7 @@ function renderRunOnceHuman(payload: Record<string, unknown>): string {
     "",
     table(["FOLLOWER", "PHASE", "OBSERVED", "TARGET", "TRIGGERED", "IN_FLIGHT", "DECISION"], rows),
     timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`,
+    reconcileRows.length === 0 ? "" : `\nRECONCILE TIMELINE\n${table(["FOLLOWER", "STEP", "STATUS", "SECONDS", "STARTED", "OBJECT"], reconcileRows)}`,
     writeRows.length === 0 ? "" : `\nSTATE WRITES\n${table(["FOLLOWER", "STATUS", "BEFORE_RV", "AFTER_RV", "PRESERVED", "EXIT", "MESSAGE"], writeRows)}`,
     "",
     "NEXT",
@@ -223,6 +227,36 @@ function timingRowsForFollower(item: Record<string, unknown>): unknown[][] {
   return rows;
 }
 
+function reconcileRowsFromRunOnce(payload: Record<string, unknown>, followers: Record<string, unknown>[]): unknown[][] {
+  const timeline = asOptionalRecord(payload.reconcileTimeline);
+  if (timeline !== null) return reconcileRowsForTimeline(timeline, null);
+  return followers.flatMap(reconcileRowsForFollower);
+}
+
+function reconcileRowsForFollower(item: Record<string, unknown>): unknown[][] {
+  return reconcileRowsForTimeline(asOptionalRecord(item.reconcileTimeline), stringOrNull(item.id));
+}
+
+function reconcileRowsForTimeline(timeline: Record<string, unknown> | null, fallbackFollower: string | null): unknown[][] {
+  if (timeline === null) return [];
+  const steps = arrayRecords(timeline.steps);
+  if (steps.length === 0 && stringOrNull(timeline.missingReason) !== null) {
+    return [[fallbackFollower ?? "-", "controller-loop", "-", "-", "-", stringOrNull(timeline.missingReason)]];
+  }
+  return steps.map((step) => [
+    stringOrNull(step.follower) ?? fallbackFollower ?? "-",
+    step.step ?? "-",
+    step.status ?? "-",
+    formatSeconds(secondsFromMs(numberOrNull(step.elapsedMs))),
+    stringOrNull(step.startedAt) ?? "-",
+    stringOrNull(step.object) ?? stringOrNull(step.pipelineRun) ?? shortSha(stringOrNull(step.observedSha)),
+  ]);
+}
+
+function secondsFromMs(value: number | null): number | null {
+  return value === null ? null : Math.round(value / 100) / 10;
+}
+
 function formatSeconds(value: number | null): string {
   return value === null ? "-" : `${value}s`;
 }