From 28f0a0c4b9a66dc5a8755a6eefefced744b4c58b Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 4 Jul 2026 01:37:45 +0000 Subject: [PATCH 1/2] feat(cicd): add branch follower reconcile timeline --- scripts/src/cicd-branch-follower.ts | 25 +++- scripts/src/cicd-reconcile-timeline.ts | 171 +++++++++++++++++++++++++ scripts/src/cicd-render.ts | 34 +++++ 3 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 scripts/src/cicd-reconcile-timeline.ts diff --git a/scripts/src/cicd-branch-follower.ts b/scripts/src/cicd-branch-follower.ts index 44caefd2..be29098e 100644 --- a/scripts/src/cicd-branch-follower.ts +++ b/scripts/src/cicd-branch-follower.ts @@ -31,6 +31,7 @@ import { invalidRuntimeReuseConfig, missingRuntimeReuseConfig, parseRuntimeReuse import { prioritizedTaskRunItems } from "./cicd-taskruns"; import { runBranchFollowerTaskRunDrillDown } from "./cicd-taskrun-drilldown"; import { runBranchFollowerJobDrillDown, runBranchFollowerRuntimeDrillDown } from "./cicd-job-runtime-drilldown"; +import { attachReconcileTimeline, compactReconcileTimeline, finishReconcileStep, finishReconcileTimeline, startReconcileStep, startReconcileTimeline } from "./cicd-reconcile-timeline"; import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types"; import { arrayField, @@ -686,17 +687,27 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions) }; } const selected = selectFollowers(registry, options, { includeDisabled: false }); + const reconcileTimeline = startReconcileTimeline({ controller: options.inCluster, dryRun: options.dryRun, confirm: options.confirm, wait: options.wait, followerIds: selected.map((follower) => follower.id) }); + const stateReadStep = startReconcileStep(reconcileTimeline, "*", "state-read"); const previous = readK8sState(registry, options); + finishReconcileStep(stateReadStep, { status: previous.ok ? "ok" : "degraded", object: registry.controller.stateConfigMapName, reason: previous.errors.join("; ") }); const results: FollowerState[] = []; const stateWriteWarnings: string[] = []; const stateWrites: Record[] = []; for (const follower of selected) { const oldState = previous.stateByFollower[follower.id] ?? {}; + const statusReadStep = startReconcileStep(reconcileTimeline, follower.id, "status-read"); const live = await readAdapterStatus(registry, follower, options); + finishReconcileStep(statusReadStep, { status: live.ok ? "ok" : "failed", observedSha: live.observedSha, targetSha: live.targetSha, phase: live.phase, pipelineRun: live.pipelineRun, message: live.message }); + const decideStep = startReconcileStep(reconcileTimeline, follower.id, "decide"); const state = await decideAndMaybeTrigger(registry, follower, oldState, live, options); + finishReconcileStep(decideStep, { status: state.phase === "Failed" || state.phase === "Blocked" ? "blocked" : "ok", observedSha: state.source.observedSha, targetSha: state.target.targetSha, phase: state.phase, pipelineRun: state.pipelineRun, message: state.decision }); if (!options.dryRun || options.recordState) { + const pendingWriteStep = startReconcileStep(reconcileTimeline, follower.id, "state-write"); + state.command = attachReconcileTimeline(state.command, reconcileTimeline, follower.id); const write = writeFollowerState(registry, state, options); - const writeSummary = stateWriteSummary(follower.id, write); + finishReconcileStep(pendingWriteStep, { status: write.exitCode === 0 ? "ok" : "failed", object: registry.controller.stateConfigMapName, exitCode: write.exitCode, reason: write.stderr || write.stdout }); + const writeSummary = stateWriteSummary(follower.id, write, pendingWriteStep.step.elapsedMs); stateWrites.push(writeSummary); if (write.exitCode !== 0) { const warning = `state write failed for ${follower.id}: ${tailText(write.stderr || write.stdout, 300)}`; @@ -706,6 +717,7 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions) } results.push(state); } + finishReconcileTimeline(reconcileTimeline); return { ok: results.every((item) => item.phase !== "Failed" && item.phase !== "Blocked"), action: "run-once", @@ -714,6 +726,7 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions) wait: options.wait, controller: options.inCluster, registry: registrySummary(registry), + reconcileTimeline: compactReconcileTimeline(reconcileTimeline), followers: results, stateWrites, warnings: stateWriteWarnings, @@ -1937,6 +1950,11 @@ function mergeFollowerStatus( const lastTriggeredSha = live?.lastTriggeredSha ?? stringOrNull(stored.lastTriggeredSha); const lastSucceededSha = live?.lastSucceededSha ?? stringOrNull(stored.lastSucceededSha); const timings = live === null ? storedFollowerTimingsForStatus(follower, asOptionalRecord(stored.timings), phase, observedSha) : buildFollowerTimings(follower, live, undefined, asOptionalRecord(stored.timings), phase); + const reconcileTimeline = compactReconcileTimeline(asOptionalRecord(stored.command)?.reconcileTimeline, follower.id) ?? { + bounded: true, + missingReason: "stored state lacks reconcileTimeline; old data cannot be reconstructed", + steps: [], + }; const summary: Record = { ok: live === null ? true : live.ok, id: follower.id, @@ -1963,6 +1981,7 @@ function mergeFollowerStatus( live: liveRequested, message: live?.message ?? stringOrNull(stored.decision) ?? "no controller state yet", timings: detailed ? timings : compactListTimings(timings), + reconcileTimeline: detailed ? reconcileTimeline : null, drilldown: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --live`, }; if (!detailed) return summary; @@ -2063,7 +2082,7 @@ function kubeConfigMapFollowerState(registry: BranchFollowerRegistry, options: P }; } -function stateWriteSummary(followerId: string, result: CommandResult): Record { +function stateWriteSummary(followerId: string, result: CommandResult, elapsedMs?: number): Record { const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; return { follower: followerId, @@ -2073,6 +2092,7 @@ function stateWriteSummary(followerId: string, result: CommandResult): Record | undefined): Reco exitCode: numberOrNull(command.exitCode), timedOut: command.timedOut === true, statusAuthority: stringOrNull(command.statusAuthority), + reconcileTimeline: compactReconcileTimeline(command.reconcileTimeline), parsedDownstreamCliOutput: false, }; } diff --git a/scripts/src/cicd-reconcile-timeline.ts b/scripts/src/cicd-reconcile-timeline.ts new file mode 100644 index 00000000..2b4a47d6 --- /dev/null +++ b/scripts/src/cicd-reconcile-timeline.ts @@ -0,0 +1,171 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower reconcile timeline visibility. +// Responsibility: bounded controller-loop timing summaries for branch-follower state/status output. + +export type ReconcileTimelineStep = { + follower: string; + step: string; + status: string; + startedAt: string; + finishedAt?: string; + elapsedMs?: number; + observedSha?: string; + targetSha?: string; + phase?: string; + pipelineRun?: string; + object?: string; + message?: string; + reason?: string; + exitCode?: number; +}; + +export type ReconcileTimeline = { + startedAt: string; + finishedAt?: string; + elapsedMs?: number; + controller: boolean; + dryRun: boolean; + confirm: boolean; + wait: boolean; + followerCount: number; + followers: string[]; + bounded: true; + steps: ReconcileTimelineStep[]; +}; + +export type ReconcileStepMarker = { + readonly step: ReconcileTimelineStep; + readonly startedMs: number; +}; + +export function startReconcileTimeline(input: { controller: boolean; dryRun: boolean; confirm: boolean; wait: boolean; followerIds: string[] }): ReconcileTimeline { + return { + startedAt: new Date().toISOString(), + controller: input.controller, + dryRun: input.dryRun, + confirm: input.confirm, + wait: input.wait, + followerCount: input.followerIds.length, + followers: input.followerIds.slice(0, 8), + bounded: true, + steps: [], + }; +} + +export function finishReconcileTimeline(timeline: ReconcileTimeline): ReconcileTimeline { + const finishedMs = Date.now(); + timeline.finishedAt = new Date(finishedMs).toISOString(); + timeline.elapsedMs = elapsedMs(timeline.startedAt, finishedMs); + timeline.steps = compactSteps(timeline.steps, null, 32); + return timeline; +} + +export function startReconcileStep(timeline: ReconcileTimeline, follower: string, step: string): ReconcileStepMarker { + const startedMs = Date.now(); + const record: ReconcileTimelineStep = { + follower: safeText(follower, 80) ?? "-", + step: safeText(step, 80) ?? "-", + status: "running", + startedAt: new Date(startedMs).toISOString(), + }; + timeline.steps.push(record); + return { step: record, startedMs }; +} + +export function finishReconcileStep(marker: ReconcileStepMarker, fields: Record = {}): ReconcileTimelineStep { + const finishedMs = Date.now(); + marker.step.finishedAt = new Date(finishedMs).toISOString(); + marker.step.elapsedMs = Math.max(0, finishedMs - marker.startedMs); + marker.step.status = safeText(fields.status, 80) ?? (fields.ok === false ? "failed" : "ok"); + setText(marker.step, "observedSha", fields.observedSha, 80); + setText(marker.step, "targetSha", fields.targetSha, 80); + setText(marker.step, "phase", fields.phase, 80); + setText(marker.step, "pipelineRun", fields.pipelineRun, 120); + setText(marker.step, "object", fields.object, 120); + setText(marker.step, "message", fields.message, 180); + setText(marker.step, "reason", fields.reason, 180); + const exitCode = numberOrNull(fields.exitCode); + if (exitCode !== null) marker.step.exitCode = exitCode; + return marker.step; +} + +export function attachReconcileTimeline(command: Record | undefined, timeline: ReconcileTimeline, followerId: string): Record | undefined { + const compact = compactReconcileTimeline(timeline, followerId); + if (compact === null) return command; + return { ...(command ?? {}), reconcileTimeline: compact }; +} + +export function compactReconcileTimeline(value: unknown, followerId?: string | null): Record | null { + const source = asRecord(value); + if (source === null) return null; + const steps = compactSteps(arrayRecords(source.steps), followerId ?? null, followerId === undefined || followerId === null ? 32 : 16); + return { + startedAt: safeText(source.startedAt, 80), + finishedAt: safeText(source.finishedAt, 80), + elapsedMs: numberOrNull(source.elapsedMs), + controller: source.controller === true, + dryRun: source.dryRun === true, + confirm: source.confirm === true, + wait: source.wait === true, + followerCount: numberOrNull(source.followerCount), + followers: Array.isArray(source.followers) ? source.followers.map((item) => safeText(item, 80)).filter((item): item is string => item !== null).slice(0, 8) : [], + bounded: true, + omittedStepCount: Math.max(0, arrayRecords(source.steps).length - steps.length), + steps, + }; +} + +function compactSteps(values: Record[], followerId: string | null, maxSteps: number): ReconcileTimelineStep[] { + const filtered = values + .filter((item) => followerId === null || item.follower === "*" || item.follower === followerId) + .slice(-maxSteps); + return filtered.map((item) => { + const step: ReconcileTimelineStep = { + follower: safeText(item.follower, 80) ?? "-", + step: safeText(item.step, 80) ?? "-", + status: safeText(item.status, 80) ?? "-", + startedAt: safeText(item.startedAt, 80) ?? "-", + }; + setText(step, "finishedAt", item.finishedAt, 80); + const elapsed = numberOrNull(item.elapsedMs); + if (elapsed !== null) step.elapsedMs = elapsed; + setText(step, "observedSha", item.observedSha, 80); + setText(step, "targetSha", item.targetSha, 80); + setText(step, "phase", item.phase, 80); + setText(step, "pipelineRun", item.pipelineRun, 120); + setText(step, "object", item.object, 120); + setText(step, "message", item.message, 180); + setText(step, "reason", item.reason, 180); + const exitCode = numberOrNull(item.exitCode); + if (exitCode !== null) step.exitCode = exitCode; + return step; + }); +} + +function setText(target: ReconcileTimelineStep, key: keyof ReconcileTimelineStep, value: unknown, maxLength: number): void { + const text = safeText(value, maxLength); + if (text !== null) (target as Record)[key] = text; +} + +function elapsedMs(startedAt: string, finishedMs: number): number | undefined { + const startedMs = Date.parse(startedAt); + if (!Number.isFinite(startedMs)) return undefined; + return Math.max(0, finishedMs - startedMs); +} + +function asRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; +} + +function arrayRecords(value: unknown): Record[] { + return Array.isArray(value) ? value.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function safeText(value: unknown, maxLength: number): string | null { + if (typeof value !== "string" || value.length === 0) return null; + const text = value.replace(/\s+/gu, " "); + return text.length <= maxLength ? text : `${text.slice(0, Math.max(0, maxLength - 3))}...`; +} diff --git a/scripts/src/cicd-render.ts b/scripts/src/cicd-render.ts index fb5831a4..3646441e 100644 --- a/scripts/src/cicd-render.ts +++ b/scripts/src/cicd-render.ts @@ -110,6 +110,7 @@ function renderStatusHuman(payload: Record, _options: ParsedOpt const next = asOptionalRecord(payload.next); const errors = Array.isArray(payload.errors) ? payload.errors : []; const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48); + const reconcileRows = followers.flatMap(reconcileRowsForFollower).slice(0, 48); return [ `CI/CD BRANCH-FOLLOWER STATUS (${payload.ok === false ? "degraded" : "ok"})`, "", @@ -120,6 +121,7 @@ function renderStatusHuman(payload: Record, _options: ParsedOpt "", table(["FOLLOWER", "PHASE", "ADAPTER", "OBSERVED", "TARGET", "TRIGGERED", "SUCCEEDED", "IN_FLIGHT", "BUDGET", "MESSAGE"], rows), timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`, + reconcileRows.length === 0 ? "" : `\nRECONCILE TIMELINE\n${table(["FOLLOWER", "STEP", "STATUS", "SECONDS", "STARTED", "OBJECT"], reconcileRows)}`, errors.length === 0 ? "" : `\nERRORS\n${errors.map((item) => `- ${item}`).join("\n")}`, "", "NEXT", @@ -147,6 +149,7 @@ function renderRunOnceHuman(payload: Record): string { }); const next = asOptionalRecord(payload.next); const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48); + const reconcileRows = reconcileRowsFromRunOnce(payload, followers).slice(0, 48); const writeRows = stateWrites.map((item) => [ item.follower, item.ok === true ? "ok" : "failed", @@ -161,6 +164,7 @@ function renderRunOnceHuman(payload: Record): string { "", table(["FOLLOWER", "PHASE", "OBSERVED", "TARGET", "TRIGGERED", "IN_FLIGHT", "DECISION"], rows), timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`, + reconcileRows.length === 0 ? "" : `\nRECONCILE TIMELINE\n${table(["FOLLOWER", "STEP", "STATUS", "SECONDS", "STARTED", "OBJECT"], reconcileRows)}`, writeRows.length === 0 ? "" : `\nSTATE WRITES\n${table(["FOLLOWER", "STATUS", "BEFORE_RV", "AFTER_RV", "PRESERVED", "EXIT", "MESSAGE"], writeRows)}`, "", "NEXT", @@ -223,6 +227,36 @@ function timingRowsForFollower(item: Record): unknown[][] { return rows; } +function reconcileRowsFromRunOnce(payload: Record, followers: Record[]): unknown[][] { + const timeline = asOptionalRecord(payload.reconcileTimeline); + if (timeline !== null) return reconcileRowsForTimeline(timeline, null); + return followers.flatMap(reconcileRowsForFollower); +} + +function reconcileRowsForFollower(item: Record): unknown[][] { + return reconcileRowsForTimeline(asOptionalRecord(item.reconcileTimeline), stringOrNull(item.id)); +} + +function reconcileRowsForTimeline(timeline: Record | null, fallbackFollower: string | null): unknown[][] { + if (timeline === null) return []; + const steps = arrayRecords(timeline.steps); + if (steps.length === 0 && stringOrNull(timeline.missingReason) !== null) { + return [[fallbackFollower ?? "-", "controller-loop", "-", "-", "-", stringOrNull(timeline.missingReason)]]; + } + return steps.map((step) => [ + stringOrNull(step.follower) ?? fallbackFollower ?? "-", + step.step ?? "-", + step.status ?? "-", + formatSeconds(secondsFromMs(numberOrNull(step.elapsedMs))), + stringOrNull(step.startedAt) ?? "-", + stringOrNull(step.object) ?? stringOrNull(step.pipelineRun) ?? shortSha(stringOrNull(step.observedSha)), + ]); +} + +function secondsFromMs(value: number | null): number | null { + return value === null ? null : Math.round(value / 100) / 10; +} + function formatSeconds(value: number | null): string { return value === null ? "-" : `${value}s`; } From 8521dcccc294b7055a7623ab9e3cde2dbf6fccdd Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 4 Jul 2026 01:43:37 +0000 Subject: [PATCH 2/2] fix(cicd): avoid persisting running state-write timeline --- scripts/src/cicd-branch-follower.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/src/cicd-branch-follower.ts b/scripts/src/cicd-branch-follower.ts index be29098e..8d4b5254 100644 --- a/scripts/src/cicd-branch-follower.ts +++ b/scripts/src/cicd-branch-follower.ts @@ -703,8 +703,8 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions) const state = await decideAndMaybeTrigger(registry, follower, oldState, live, options); finishReconcileStep(decideStep, { status: state.phase === "Failed" || state.phase === "Blocked" ? "blocked" : "ok", observedSha: state.source.observedSha, targetSha: state.target.targetSha, phase: state.phase, pipelineRun: state.pipelineRun, message: state.decision }); if (!options.dryRun || options.recordState) { - const pendingWriteStep = startReconcileStep(reconcileTimeline, follower.id, "state-write"); state.command = attachReconcileTimeline(state.command, reconcileTimeline, follower.id); + const pendingWriteStep = startReconcileStep(reconcileTimeline, follower.id, "state-write"); const write = writeFollowerState(registry, state, options); finishReconcileStep(pendingWriteStep, { status: write.exitCode === 0 ? "ok" : "failed", object: registry.controller.stateConfigMapName, exitCode: write.exitCode, reason: write.stderr || write.stdout }); const writeSummary = stateWriteSummary(follower.id, write, pendingWriteStep.step.elapsedMs);