diff --git a/scripts/src/cicd-branch-follower.ts b/scripts/src/cicd-branch-follower.ts index 6ac94dae..27f9c974 100644 --- a/scripts/src/cicd-branch-follower.ts +++ b/scripts/src/cicd-branch-follower.ts @@ -30,14 +30,14 @@ import { compactRefreshEvidence, followerEvidenceSummary } from "./cicd-evidence import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native"; import { argoApplicationReady, nativeArgoSummary, nativeGitMirrorReady, nativeGitMirrorRequired, nativeGitMirrorSummary, nativePipelineRunSummary, nativeRuntimeSummary, pipelineRunSucceeded, runtimeTargetShaFromWorkloads, runtimeWorkloadsReady } from "./cicd-native-summary"; import { invalidRuntimeReuseConfig, missingRuntimeReuseConfig, parseRuntimeReuseConfig, requiredReuseServiceError, RUNTIME_REUSE_CONFIG_PATH, summarizeRuntimeReuseConfig, type RuntimeReuseConfig } from "./cicd-reuse-config"; -import { prioritizedTaskRunItems } from "./cicd-taskruns"; import { runBranchFollowerTaskRunDrillDown } from "./cicd-taskrun-drilldown"; import { runBranchFollowerJobDrillDown, runBranchFollowerRuntimeDrillDown } from "./cicd-job-runtime-drilldown"; import { runBranchFollowerGate } from "./cicd-gates"; import { buildCicdHelp } from "./cicd-help"; import { attachReconcileTimeline, compactReconcileTimeline, finishReconcileStep, finishReconcileTimeline, startReconcileStep, startReconcileTimeline } from "./cicd-reconcile-timeline"; import { orderFollowersForControllerCloseout, shouldYieldAfterAutomaticTrigger } from "./cicd-reconcile-scheduler"; -import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerGate, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types"; +import { buildFollowerTimings, compactListTimings, compactTimings, storedFollowerTimingsForStatus, timingPerformanceSummary } from "./cicd-timings"; +import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerGate, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, TriggerResult } from "./cicd-types"; import { arrayField, asRecord, @@ -633,7 +633,7 @@ async function buildStatus(registry: BranchFollowerRegistry, options: ParsedOpti const shouldLive = wantsLive && options.inCluster; const selected = selectFollowers(registry, options, { includeDisabled: true }); const followers = []; - const detailedFollowers = options.followerId !== null || options.full; + const detailedFollowers = options.full; for (const follower of selected) { const stored = k8s.stateByFollower[follower.id] ?? {}; const fallbackStored = refresh === null ? {} : beforeRefreshStateByFollower[follower.id] ?? {}; @@ -1995,10 +1995,12 @@ function mergeFollowerStatus( lastSucceededSha, pipelineRun: live?.pipelineRun ?? stringOrNull(stored.pipelineRun), inFlightJob: live?.inFlightJob ?? stringOrNull(stored.inFlightJob), + budgetSource: follower.budgets, updatedAt: stringOrNull(stored.updatedAt), live: liveRequested, message: live?.message ?? stringOrNull(stored.decision) ?? "no controller state yet", timings: detailed ? timings : compactListTimings(timings), + performance: timingPerformanceSummary(timings), evidence: detailed ? evidence : null, reconcileTimeline: detailed ? reconcileTimeline : null, rawStateDiagnostic: detailed ? asOptionalRecord(stored.rawStateDiagnostic) : null, @@ -2011,28 +2013,12 @@ function mergeFollowerStatus( ...asOptionalRecord(summary.source), snapshotPrefix: follower.source.snapshotPrefix, }, - budgetSource: follower.budgets, stateConfigMap: registry.controller.stateConfigMapName, warnings: Array.isArray(stored.warnings) ? stored.warnings.slice(0, 6) : [], next: followerNextCommands(follower), }; } -function compactListTimings(timings: FollowerState["timings"]): Record { - return { - budgetSeconds: timings.budgetSeconds, - totalSeconds: timings.totalSeconds, - totalStatus: timings.totalStatus, - sourceCommit: timings.sourceCommit, - overBudget: timings.overBudget, - stages: timings.stages.slice(0, 4).map((stage) => ({ - stage: stage.stage, - status: stage.status, - seconds: stage.seconds, - })), - }; -} - function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead { const errors: string[] = []; const stateResult = kubeConfigMapFollowerState(registry, options); @@ -2405,300 +2391,6 @@ function compactSourcePayload(source: Record | null): Record | undefined, - storedTimings?: Record | null, - phase?: BranchFollowerPhase, -): FollowerState["timings"] { - const nativePayload = asOptionalRecord(live.payload); - const finishOverride = stringOrNull(triggerCommand?.finishedAt) ?? noopStoredTotalFinishOverride(storedTimings, phase, live); - const total = totalTimingFromCommand(triggerCommand, phase) ?? totalTimingFromStored(storedTimings, phase, finishOverride, live.observedSha); - const storedStages = live.observedSha !== null && stringOrNull(storedTimings?.sourceCommit) === live.observedSha ? storedStageTimings(storedTimings ?? null) : []; - const stages = dedupeTimingStages([ - ...stageTimingsFromCommand(triggerCommand), - ...stageTimingsFromNativePayload(nativePayload), - ...storedStages, - ]).slice(0, 24); - const stageSourceCommit = stages.length > 0 ? live.observedSha : null; - return { - budgetSeconds: follower.budgets.endToEndSeconds, - totalSeconds: total?.seconds ?? null, - totalStatus: total?.status ?? "unknown", - totalSource: total?.source ?? "-", - sourceCommit: total?.sourceCommit ?? stringOrNull(triggerCommand?.sourceCommit) ?? stageSourceCommit, - startedAt: total?.startedAt ?? null, - finishedAt: total?.finishedAt ?? null, - overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, - stages, - }; -} - -function noopStoredTotalFinishOverride( - storedTimings: Record | null | undefined, - phase: BranchFollowerPhase | undefined, - live: AdapterSummary, -): string | null { - if (phase !== "Noop" || live.aligned !== true || live.observedSha === null) return null; - if (storedTimings === null || storedTimings === undefined) return null; - if (stringOrNull(storedTimings.sourceCommit) !== live.observedSha) return null; - if (stringOrNull(storedTimings.startedAt) === null) return null; - if (stringOrNull(storedTimings.finishedAt) !== null) return null; - return new Date().toISOString(); -} - -function storedFollowerTimingsForStatus( - follower: FollowerSpec, - storedTimings: Record | null, - phase: BranchFollowerPhase, - observedSha: string | null, -): FollowerState["timings"] { - const total = totalTimingFromStored(storedTimings, phase, null, observedSha); - const sourceCommit = total?.sourceCommit ?? stringOrNull(storedTimings?.sourceCommit) ?? null; - return { - budgetSeconds: follower.budgets.endToEndSeconds, - totalSeconds: total?.seconds ?? null, - totalStatus: total?.status ?? "unknown", - totalSource: total?.source ?? "-", - sourceCommit, - startedAt: total?.startedAt ?? null, - finishedAt: total?.finishedAt ?? null, - overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, - stages: sourceCommit === null ? [] : storedStageTimings(storedTimings), - }; -} - -function storedStageTimings(storedTimings: Record | null): StageTiming[] { - if (storedTimings === null) return []; - return arrayRecords(storedTimings.stages) - .map((stage) => stageTiming( - stringOrNull(stage.stage) ?? "", - stringOrNull(stage.status) ?? "unknown", - numberOrNull(stage.seconds), - numberOrNull(stage.budgetSeconds), - stringOrNull(stage.source) ?? "stored-state", - stringOrNull(stage.object), - )) - .filter((stage) => stage.stage.length > 0) - .slice(0, 24); -} - -function compactTimings(timings: FollowerState["timings"]): FollowerState["timings"] { - return { - budgetSeconds: timings.budgetSeconds, - totalSeconds: timings.totalSeconds, - totalStatus: timings.totalStatus, - totalSource: timings.totalSource, - sourceCommit: timings.sourceCommit, - startedAt: timings.startedAt, - finishedAt: timings.finishedAt, - overBudget: timings.overBudget, - stages: timings.stages.slice(0, 24).map((stage) => ({ - stage: stage.stage, - status: stage.status, - seconds: stage.seconds, - budgetSeconds: stage.budgetSeconds, - source: stage.source, - object: stage.object, - })), - }; -} - -function totalTimingFromCommand(command: Record | undefined, phase?: BranchFollowerPhase): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null { - if (command === undefined) return null; - if (command.mode === "k8s-native-closeout") return null; - const payload = asOptionalRecord(command.payload); - if (payload?.reused === true) return null; - const startedAt = stringOrNull(command.startedAt); - const finishedAt = stringOrNull(command.finishedAt); - const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? secondsFromMs(numberOrNull(command.elapsedMs)); - if (seconds === null) return null; - const closeout = asOptionalRecord(command.closeout); - const exitCode = numberOrNull(command.exitCode); - const status = command.ok === false || (exitCode !== null && exitCode !== 0) - ? "failed" - : command.budgetTimedOut === true || closeout?.timedOut === true - ? "over-budget" - : command.timedOut === true - ? "timed-out" - : closeout?.completed === true || command.completed === true - ? "completed" - : command.stillRunning === true - ? "running" - : command.pipelineRunCompleted === true - ? "ci-completed" - : phase === undefined - ? "submitted" - : phase.toLowerCase(); - return { seconds, status, source: stringOrNull(command.mode) ?? stringOrNull(command.status) ?? "command", sourceCommit: stringOrNull(command.sourceCommit), startedAt, finishedAt }; -} - -function totalTimingFromStored(storedTimings: Record | null | undefined, phase?: BranchFollowerPhase, finishOverride?: string | null, observedSha?: string | null): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null { - if (storedTimings === null || storedTimings === undefined) return null; - const status = stringOrNull(storedTimings.totalStatus); - const source = stringOrNull(storedTimings.totalSource); - const sourceCommit = stringOrNull(storedTimings.sourceCommit); - if (sourceCommit === null) return null; - if (observedSha !== null && observedSha !== undefined && sourceCommit !== observedSha) return null; - const startedAt = stringOrNull(storedTimings.startedAt); - const finishedAt = stringOrNull(storedTimings.finishedAt) ?? finishOverride ?? null; - if (phase === "Noop" && finishedAt === null) return null; - const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? numberOrNull(storedTimings.totalSeconds); - if (seconds === null) return null; - return { - seconds, - status: finishedAt === null && phase !== undefined && !terminalPhase(phase) ? phase.toLowerCase() : phase === undefined ? status ?? "recorded" : phase.toLowerCase(), - source: source ?? "stored-state", - sourceCommit, - startedAt, - finishedAt, - }; -} - -function totalSecondsFromRange(startedAt: string | null, finishedAt: string | null): number | null { - const startedMs = timestampMs(startedAt); - if (startedMs === null) return null; - const finishedMs = timestampMs(finishedAt) ?? Date.now(); - return finishedMs >= startedMs ? roundSeconds((finishedMs - startedMs) / 1000) : null; -} - -function timestampMs(value: string | null): number | null { - if (value === null) return null; - const parsed = Date.parse(value); - return Number.isFinite(parsed) ? parsed : null; -} - -function terminalPhase(phase: BranchFollowerPhase): boolean { - return phase === "Succeeded" || phase === "Failed" || phase === "Blocked" || phase === "Skipped" || phase === "Noop"; -} - -function stageTimingsFromNativePayload(payload: Record | null): StageTiming[] { - if (payload === null) return []; - const stages: StageTiming[] = []; - const statusRead = asOptionalRecord(asOptionalRecord(payload.timings)?.statusRead); - stages.push(stageTiming("status-read", "ok", secondsFromMs(numberOrNull(statusRead?.elapsedMs)), numberOrNull(statusRead?.budgetSeconds), "native-status", null)); - const sourceSyncStage = k8sJobTiming("git-mirror-sync", asOptionalRecord(payload.sourceSync)); - if (sourceSyncStage !== null) stages.push(sourceSyncStage); - const reuseConfig = asOptionalRecord(payload.reuseConfig); - if (reuseConfig !== null) { - stages.push(stageTiming("reuse-config", reuseConfig.ok === true ? "ready" : "missing-or-invalid", null, null, "source-gitops", stringOrNull(reuseConfig.path))); - } - const gitMirror = asOptionalRecord(payload.gitMirror); - if (gitMirror !== null) { - const hasGitopsBranch = stringOrNull(gitMirror.gitopsBranch) !== null; - const sourceReady = gitMirror.sourceSnapshotReady === true; - const status = gitMirror.pendingFlush === true - ? "pending-flush" - : hasGitopsBranch - ? gitMirror.githubInSync === true && sourceReady ? "ready" : "not-ready" - : sourceReady ? "source-ready" : "source-not-ready"; - stages.push(stageTiming("git-mirror", status, null, null, "git-mirror-cache", stringOrNull(gitMirror.gitopsBranch) ?? stringOrNull(gitMirror.sourceBranch))); - } - const tekton = asOptionalRecord(payload.tekton); - if (tekton !== null) { - const status = tekton.succeeded === true ? "succeeded" : tekton.succeeded === false ? `failed:${stringOrNull(tekton.reason) ?? "unknown"}` : "running"; - stages.push(stageTiming("pipelinerun", status, numberOrNull(tekton.durationSeconds), null, "tekton", stringOrNull(tekton.name))); - } - const taskRuns = asOptionalRecord(payload.taskRuns); - for (const record of taskRuns === null ? [] : prioritizedTaskRunItems(taskRuns)) { - const name = stringOrNull(record.pipelineTask) ?? stringOrNull(record.name) ?? "unknown"; - const status = record.status === "True" ? "succeeded" : record.status === "False" ? `failed:${stringOrNull(record.reason) ?? "unknown"}` : "running"; - stages.push(stageTiming(`task:${name}`, status, numberOrNull(record.durationSeconds), null, "tekton-taskrun", stringOrNull(record.name))); - } - const argo = asOptionalRecord(payload.argo); - if (argo !== null) { - stages.push(stageTiming("argo", `${stringOrNull(argo.syncStatus) ?? "unknown"}/${stringOrNull(argo.healthStatus) ?? "unknown"}`, numberOrNull(argo.operationDurationSeconds), null, "argocd", stringOrNull(argo.name))); - } - const runtime = asOptionalRecord(payload.runtime); - if (runtime !== null) { - const aligned = runtime.aligned === true ? "aligned" : runtime.aligned === false ? "stale" : "unknown-target"; - stages.push(stageTiming("runtime", `${runtime.ready === true ? "ready" : "not-ready"}/${aligned}`, null, null, "kubernetes-workload", stringOrNull(runtime.namespace))); - } - return stages; -} - -function stageTimingsFromCommand(command: Record | undefined): StageTiming[] { - if (command === undefined) return []; - const stages: StageTiming[] = []; - const phase = stringOrNull(command.phase); - const jobStage = phase === null ? null : k8sJobTiming(phase, asOptionalRecord(command.job), stringOrNull(command.jobName)); - if (jobStage !== null) stages.push(jobStage); - const payload = asOptionalRecord(command.payload); - if (payload !== null) { - const capabilities = asOptionalRecord(payload.nativeCapabilities); - for (const stage of [ - k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)), - k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)), - k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)), - ]) { - if (stage !== null) stages.push(stage); - } - const agentrun = asOptionalRecord(payload.agentrun); - const agentrunSync = asOptionalRecord(agentrun?.gitMirrorSync); - const agentrunFlush = asOptionalRecord(agentrun?.gitMirrorFlush); - const imageBuild = asOptionalRecord(agentrun?.imageBuild); - const gitopsPublish = asOptionalRecord(agentrun?.gitopsPublish); - for (const stage of [ - k8sJobTiming("git-mirror-sync", asOptionalRecord(agentrunSync?.payload), stringOrNull(agentrunSync?.jobName)), - k8sJobTiming("image-build", asOptionalRecord(imageBuild?.result), stringOrNull(imageBuild?.jobName)), - k8sJobTiming("gitops-publish", asOptionalRecord(gitopsPublish?.result), stringOrNull(gitopsPublish?.jobName)), - k8sJobTiming("git-mirror-flush", asOptionalRecord(agentrunFlush?.payload), stringOrNull(agentrunFlush?.jobName)), - ]) { - if (stage !== null) stages.push(stage); - } - const tektonSeconds = secondsFromMs(numberOrNull(payload.elapsedMs)); - if (tektonSeconds !== null) { - const status = payload.completed === true ? "completed" : payload.failed === true ? "failed" : payload.stillRunning === true ? "running" : "submitted"; - stages.push(stageTiming("pipelinerun-wait", status, tektonSeconds, null, "tekton-submit", stringOrNull(command.pipelineRun))); - } - } - const closeout = asOptionalRecord(command.closeout); - if (closeout !== null) { - const gitMirrorFlush = asOptionalRecord(closeout.gitMirrorFlush); - const gitMirrorFlushStage = k8sJobTiming("git-mirror-flush", asOptionalRecord(gitMirrorFlush?.result), stringOrNull(gitMirrorFlush?.jobName)); - if (gitMirrorFlushStage !== null) stages.push(gitMirrorFlushStage); - const status = closeout.completed === true ? "completed" : closeout.timedOut === true ? "over-budget" : "pending"; - stages.push(stageTiming("closeout", status, secondsFromMs(numberOrNull(closeout.elapsedMs)), null, "k8s-native-closeout", stringOrNull(command.pipelineRun))); - } - return stages; -} - -function k8sJobTiming(stage: string, job: Record | null, objectOverride?: string | null): StageTiming | null { - if (job === null) return null; - const status = job.completed === true - ? job.reused === true ? "reused" : "completed" - : job.failed === true - ? "failed" - : job.timedOut === true - ? "over-budget" - : "running"; - return stageTiming(stage, status, secondsFromMs(numberOrNull(job.elapsedMs)), null, "kubernetes-job", objectOverride ?? stringOrNull(job.jobName)); -} - -function stageTiming(stage: string, status: string, seconds: number | null, budgetSeconds: number | null, source: string, object: string | null): StageTiming { - return { stage, status, seconds, budgetSeconds, source, object }; -} - -function dedupeTimingStages(stages: StageTiming[]): StageTiming[] { - const byKey = new Map(); - for (const stage of stages) { - if (stage.stage.length === 0) continue; - const key = `${stage.stage}\t${stage.object ?? ""}`; - const previous = byKey.get(key); - if (previous === undefined || previous.seconds === null && stage.seconds !== null) byKey.set(key, stage); - } - return [...byKey.values()]; -} - -function secondsFromMs(value: number | null): number | null { - return value === null ? null : roundSeconds(value / 1000); -} - -function roundSeconds(value: number): number { - return Math.round(value * 10) / 10; -} - function writeFollowerState(registry: BranchFollowerRegistry, state: FollowerState, options: ParsedOptions): CommandResult { const stateJson = JSON.stringify(compactFollowerStateForConfigMap(state)); const script = [ diff --git a/scripts/src/cicd-evidence.ts b/scripts/src/cicd-evidence.ts index e59a4bae..3c86c01e 100644 --- a/scripts/src/cicd-evidence.ts +++ b/scripts/src/cicd-evidence.ts @@ -43,7 +43,7 @@ export function followerEvidenceSummary(input: { const refreshSourceCommit = stringOrNull(refresh?.sourceCommit); return { pipelineRunRefName: pipelineRefName, - pipeline, + pipeline: compactPipelineEvidence(pipeline), refreshBoundedReason: refresh === null ? "missing-from-live-and-stored-evidence" : null, refresh: refresh === null ? null @@ -57,6 +57,18 @@ export function followerEvidenceSummary(input: { }; } +function compactPipelineEvidence(value: Record | null): Record | null { + if (value === null) return null; + const metadata = asOptionalRecord(value.metadata); + const spec = asOptionalRecord(value.spec); + return { + metadata: { name: stringOrNull(metadata?.name) }, + spec: { + runtimeReadyTask: compactRefreshRuntimeReady(asOptionalRecord(spec?.runtimeReadyTask)), + }, + }; +} + function compactRefreshRender(value: Record | null): Record | null { if (value === null) return null; return { diff --git a/scripts/src/cicd-render.ts b/scripts/src/cicd-render.ts index c3370e7f..2264a4d7 100644 --- a/scripts/src/cicd-render.ts +++ b/scripts/src/cicd-render.ts @@ -110,6 +110,7 @@ function renderStatusHuman(payload: Record, _options: ParsedOpt const next = asOptionalRecord(payload.next); const errors = Array.isArray(payload.errors) ? payload.errors : []; const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48); + const performanceRows = followers.flatMap(performanceRowsForFollower).slice(0, 24); const evidenceRows = followers.flatMap(evidenceRowsForFollower).slice(0, 48); const reconcileRows = followers.flatMap(reconcileRowsForFollower).slice(0, 48); const rawStateRows = followers.flatMap(rawStateRowsForFollower).slice(0, 24); @@ -123,6 +124,7 @@ function renderStatusHuman(payload: Record, _options: ParsedOpt "", table(["FOLLOWER", "PHASE", "ADAPTER", "OBSERVED", "TARGET", "TRIGGERED", "SUCCEEDED", "IN_FLIGHT", "BUDGET", "MESSAGE"], rows), timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`, + performanceRows.length === 0 ? "" : `\nSLOW STAGES\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "SOURCE", "OBJECT"], performanceRows)}`, evidenceRows.length === 0 ? "" : `\nEVIDENCE\n${table(["FOLLOWER", "TYPE", "STATUS", "DETAIL", "OBJECT"], evidenceRows)}`, reconcileRows.length === 0 ? "" : `\nRECONCILE TIMELINE\n${table(["FOLLOWER", "STEP", "STATUS", "SECONDS", "STARTED", "OBJECT"], reconcileRows)}`, rawStateRows.length === 0 ? "" : `\nRAW STATE DIAGNOSTIC\n${table(["FOLLOWER", "STATE_BYTES", "COMMAND", "TIMELINE", "STEPS", "TIMELINE_BYTES", "REASON"], rawStateRows)}`, @@ -231,6 +233,19 @@ function timingRowsForFollower(item: Record): unknown[][] { return rows; } +function performanceRowsForFollower(item: Record): unknown[][] { + const performance = asOptionalRecord(item.performance); + if (performance === null) return []; + return arrayRecords(performance.slowStages).map((stage) => [ + item.id, + stage.stage ?? "-", + stage.status ?? "-", + formatSeconds(numberOrNull(stage.seconds)), + stringOrNull(stage.source) ?? "-", + stringOrNull(stage.object) ?? "-", + ]); +} + function reconcileRowsFromRunOnce(payload: Record, followers: Record[]): unknown[][] { const timeline = asOptionalRecord(payload.reconcileTimeline); if (timeline !== null) return reconcileRowsForTimeline(timeline, null); diff --git a/scripts/src/cicd-timings.ts b/scripts/src/cicd-timings.ts new file mode 100644 index 00000000..f7c845f6 --- /dev/null +++ b/scripts/src/cicd-timings.ts @@ -0,0 +1,355 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower timing helpers. +// Responsibility: compact branch-follower total/stage timing contracts and performance summaries. +import { prioritizedTaskRunItems } from "./cicd-taskruns"; +import type { AdapterSummary, BranchFollowerPhase, FollowerSpec, FollowerState, StageTiming } from "./cicd-types"; + +type TotalTiming = { + seconds: number; + status: string; + source: string; + sourceCommit: string | null; + startedAt: string | null; + finishedAt: string | null; +}; + +export function buildFollowerTimings( + follower: FollowerSpec, + live: AdapterSummary, + triggerCommand: Record | undefined, + storedTimings?: Record | null, + phase?: BranchFollowerPhase, +): FollowerState["timings"] { + const nativePayload = asOptionalRecord(live.payload); + const finishOverride = stringOrNull(triggerCommand?.finishedAt) ?? noopStoredTotalFinishOverride(storedTimings, phase, live); + const total = totalTimingFromCommand(triggerCommand, phase) ?? totalTimingFromStored(storedTimings, phase, finishOverride, live.observedSha); + const storedStages = live.observedSha !== null && stringOrNull(storedTimings?.sourceCommit) === live.observedSha ? storedStageTimings(storedTimings ?? null) : []; + const stages = dedupeTimingStages([ + ...stageTimingsFromCommand(triggerCommand), + ...stageTimingsFromNativePayload(nativePayload, total), + ...storedStages, + ]).slice(0, 24); + const stageSourceCommit = stages.length > 0 ? live.observedSha : null; + return { + budgetSeconds: follower.budgets.endToEndSeconds, + totalSeconds: total?.seconds ?? null, + totalStatus: total?.status ?? "unknown", + totalSource: total?.source ?? "-", + sourceCommit: total?.sourceCommit ?? stringOrNull(triggerCommand?.sourceCommit) ?? stageSourceCommit, + startedAt: total?.startedAt ?? null, + finishedAt: total?.finishedAt ?? null, + overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, + stages, + }; +} + +export function storedFollowerTimingsForStatus( + follower: FollowerSpec, + storedTimings: Record | null, + phase: BranchFollowerPhase, + observedSha: string | null, +): FollowerState["timings"] { + const total = totalTimingFromStored(storedTimings, phase, null, observedSha); + const sourceCommit = total?.sourceCommit ?? stringOrNull(storedTimings?.sourceCommit) ?? null; + return { + budgetSeconds: follower.budgets.endToEndSeconds, + totalSeconds: total?.seconds ?? null, + totalStatus: total?.status ?? "unknown", + totalSource: total?.source ?? "-", + sourceCommit, + startedAt: total?.startedAt ?? null, + finishedAt: total?.finishedAt ?? null, + overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, + stages: sourceCommit === null ? [] : storedStageTimings(storedTimings), + }; +} + +export function compactTimings(timings: FollowerState["timings"]): FollowerState["timings"] { + return { + budgetSeconds: timings.budgetSeconds, + totalSeconds: timings.totalSeconds, + totalStatus: timings.totalStatus, + totalSource: timings.totalSource, + sourceCommit: timings.sourceCommit, + startedAt: timings.startedAt, + finishedAt: timings.finishedAt, + overBudget: timings.overBudget, + stages: timings.stages.slice(0, 24).map((stage) => ({ + stage: stage.stage, + status: stage.status, + seconds: stage.seconds, + budgetSeconds: stage.budgetSeconds, + source: stage.source, + object: stage.object, + })), + }; +} + +export function compactListTimings(timings: FollowerState["timings"]): Record { + return { + budgetSeconds: timings.budgetSeconds, + totalSeconds: timings.totalSeconds, + totalStatus: timings.totalStatus, + sourceCommit: timings.sourceCommit, + overBudget: timings.overBudget, + stages: timings.stages.slice(0, 4).map((stage) => ({ + stage: stage.stage, + status: stage.status, + seconds: stage.seconds, + })), + }; +} + +export function timingPerformanceSummary(timings: FollowerState["timings"]): Record { + const slowStages = timings.stages + .filter((stage) => stage.seconds !== null) + .sort((a, b) => (b.seconds ?? 0) - (a.seconds ?? 0)) + .slice(0, 5) + .map((stage) => ({ + stage: stage.stage, + status: stage.status, + seconds: stage.seconds, + source: stage.source, + object: stage.object, + })); + return { + budgetSeconds: timings.budgetSeconds, + totalSeconds: timings.totalSeconds, + overBudget: timings.overBudget, + slowestStage: slowStages[0] ?? null, + slowStages, + note: timings.overBudget === true ? "advisory budget exceeded; inspect slowStages and stage source before rerun" : null, + }; +} + +function noopStoredTotalFinishOverride( + storedTimings: Record | null | undefined, + phase: BranchFollowerPhase | undefined, + live: AdapterSummary, +): string | null { + if (phase !== "Noop" || live.aligned !== true || live.observedSha === null) return null; + if (storedTimings === null || storedTimings === undefined) return null; + if (stringOrNull(storedTimings.sourceCommit) !== live.observedSha) return null; + if (stringOrNull(storedTimings.startedAt) === null) return null; + if (stringOrNull(storedTimings.finishedAt) !== null) return null; + return new Date().toISOString(); +} + +function storedStageTimings(storedTimings: Record | null): StageTiming[] { + if (storedTimings === null) return []; + return arrayRecords(storedTimings.stages) + .map((stage) => stageTiming( + stringOrNull(stage.stage) ?? "", + stringOrNull(stage.status) ?? "unknown", + numberOrNull(stage.seconds), + numberOrNull(stage.budgetSeconds), + stringOrNull(stage.source) ?? "stored-state", + stringOrNull(stage.object), + )) + .filter((stage) => stage.stage.length > 0) + .slice(0, 24); +} + +function totalTimingFromCommand(command: Record | undefined, phase?: BranchFollowerPhase): TotalTiming | null { + if (command === undefined) return null; + if (command.mode === "k8s-native-closeout") return null; + const payload = asOptionalRecord(command.payload); + if (payload?.reused === true) return null; + const startedAt = stringOrNull(command.startedAt); + const finishedAt = stringOrNull(command.finishedAt); + const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? secondsFromMs(numberOrNull(command.elapsedMs)); + if (seconds === null) return null; + const closeout = asOptionalRecord(command.closeout); + const exitCode = numberOrNull(command.exitCode); + const status = command.ok === false || (exitCode !== null && exitCode !== 0) + ? "failed" + : command.budgetTimedOut === true || closeout?.timedOut === true + ? "over-budget" + : command.timedOut === true + ? "timed-out" + : closeout?.completed === true || command.completed === true + ? "completed" + : command.stillRunning === true + ? "running" + : command.pipelineRunCompleted === true + ? "ci-completed" + : phase === undefined + ? "submitted" + : phase.toLowerCase(); + return { seconds, status, source: stringOrNull(command.mode) ?? stringOrNull(command.status) ?? "command", sourceCommit: stringOrNull(command.sourceCommit), startedAt, finishedAt }; +} + +function totalTimingFromStored(storedTimings: Record | null | undefined, phase?: BranchFollowerPhase, finishOverride?: string | null, observedSha?: string | null): TotalTiming | null { + if (storedTimings === null || storedTimings === undefined) return null; + const status = stringOrNull(storedTimings.totalStatus); + const source = stringOrNull(storedTimings.totalSource); + const sourceCommit = stringOrNull(storedTimings.sourceCommit); + if (sourceCommit === null) return null; + if (observedSha !== null && observedSha !== undefined && sourceCommit !== observedSha) return null; + const startedAt = stringOrNull(storedTimings.startedAt); + const finishedAt = stringOrNull(storedTimings.finishedAt) ?? finishOverride ?? null; + if (phase === "Noop" && finishedAt === null) return null; + const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? numberOrNull(storedTimings.totalSeconds); + if (seconds === null) return null; + return { + seconds, + status: finishedAt === null && phase !== undefined && !terminalPhase(phase) ? phase.toLowerCase() : phase === undefined ? status ?? "recorded" : phase.toLowerCase(), + source: source ?? "stored-state", + sourceCommit, + startedAt, + finishedAt, + }; +} + +function totalSecondsFromRange(startedAt: string | null, finishedAt: string | null): number | null { + const startedMs = timestampMs(startedAt); + if (startedMs === null) return null; + const finishedMs = timestampMs(finishedAt) ?? Date.now(); + return finishedMs >= startedMs ? roundSeconds((finishedMs - startedMs) / 1000) : null; +} + +function terminalPhase(phase: BranchFollowerPhase): boolean { + return phase === "Succeeded" || phase === "Failed" || phase === "Blocked" || phase === "Skipped" || phase === "Noop"; +} + +function stageTimingsFromNativePayload(payload: Record | null, total: TotalTiming | null): StageTiming[] { + if (payload === null) return []; + const stages: StageTiming[] = []; + const statusRead = asOptionalRecord(asOptionalRecord(payload.timings)?.statusRead); + stages.push(stageTiming("status-read", "ok", secondsFromMs(numberOrNull(statusRead?.elapsedMs)), numberOrNull(statusRead?.budgetSeconds), "native-status", null)); + const sourceSyncStage = k8sJobTiming("git-mirror-sync", asOptionalRecord(payload.sourceSync)); + if (sourceSyncStage !== null) stages.push(sourceSyncStage); + const reuseConfig = asOptionalRecord(payload.reuseConfig); + if (reuseConfig !== null) stages.push(stageTiming("reuse-config", reuseConfig.ok === true ? "ready" : "missing-or-invalid", null, null, "source-gitops", stringOrNull(reuseConfig.path))); + const gitMirror = asOptionalRecord(payload.gitMirror); + if (gitMirror !== null) { + const hasGitopsBranch = stringOrNull(gitMirror.gitopsBranch) !== null; + const sourceReady = gitMirror.sourceSnapshotReady === true; + const status = gitMirror.pendingFlush === true ? "pending-flush" : hasGitopsBranch ? gitMirror.githubInSync === true && sourceReady ? "ready" : "not-ready" : sourceReady ? "source-ready" : "source-not-ready"; + stages.push(stageTiming("git-mirror", status, null, null, "git-mirror-cache", stringOrNull(gitMirror.gitopsBranch) ?? stringOrNull(gitMirror.sourceBranch))); + } + const tekton = asOptionalRecord(payload.tekton); + if (tekton !== null) { + const status = tekton.succeeded === true ? "succeeded" : tekton.succeeded === false ? `failed:${stringOrNull(tekton.reason) ?? "unknown"}` : "running"; + stages.push(stageTiming("pipelinerun", status, numberOrNull(tekton.durationSeconds), null, "tekton", stringOrNull(tekton.name))); + } + const taskRuns = asOptionalRecord(payload.taskRuns); + for (const record of taskRuns === null ? [] : prioritizedTaskRunItems(taskRuns)) { + const name = stringOrNull(record.pipelineTask) ?? stringOrNull(record.name) ?? "unknown"; + const status = record.status === "True" ? "succeeded" : record.status === "False" ? `failed:${stringOrNull(record.reason) ?? "unknown"}` : "running"; + stages.push(stageTiming(`task:${name}`, status, numberOrNull(record.durationSeconds), null, "tekton-taskrun", stringOrNull(record.name))); + } + const argo = asOptionalRecord(payload.argo); + if (argo !== null) { + const sameWindow = total === null || timingOverlapsTotal(total, stringOrNull(argo.operationStartedAt), stringOrNull(argo.operationFinishedAt)); + stages.push(stageTiming("argo", `${sameWindow ? "" : "current:"}${stringOrNull(argo.syncStatus) ?? "unknown"}/${stringOrNull(argo.healthStatus) ?? "unknown"}`, sameWindow ? numberOrNull(argo.operationDurationSeconds) : null, null, sameWindow ? "argocd" : "argocd-current", stringOrNull(argo.name))); + } + const runtime = asOptionalRecord(payload.runtime); + if (runtime !== null) { + const aligned = runtime.aligned === true ? "aligned" : runtime.aligned === false ? "stale" : "unknown-target"; + stages.push(stageTiming("runtime", `${runtime.ready === true ? "ready" : "not-ready"}/${aligned}`, null, null, "kubernetes-workload", stringOrNull(runtime.namespace))); + } + return stages; +} + +function timingOverlapsTotal(total: TotalTiming, startedAt: string | null, finishedAt: string | null): boolean { + const totalStart = timestampMs(total.startedAt); + const totalFinish = timestampMs(total.finishedAt); + const stageStart = timestampMs(startedAt); + const stageFinish = timestampMs(finishedAt); + if (totalStart === null || totalFinish === null || stageStart === null || stageFinish === null) return true; + const slackMs = 30_000; + return stageStart <= totalFinish + slackMs && stageFinish >= totalStart - slackMs; +} + +function stageTimingsFromCommand(command: Record | undefined): StageTiming[] { + if (command === undefined) return []; + const stages: StageTiming[] = []; + const phase = stringOrNull(command.phase); + const jobStage = phase === null ? null : k8sJobTiming(phase, asOptionalRecord(command.job), stringOrNull(command.jobName)); + if (jobStage !== null) stages.push(jobStage); + const payload = asOptionalRecord(command.payload); + if (payload !== null) { + const capabilities = asOptionalRecord(payload.nativeCapabilities); + for (const stage of [ + k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)), + k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)), + k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)), + ]) if (stage !== null) stages.push(stage); + const agentrun = asOptionalRecord(payload.agentrun); + const agentrunSync = asOptionalRecord(agentrun?.gitMirrorSync); + const agentrunFlush = asOptionalRecord(agentrun?.gitMirrorFlush); + const imageBuild = asOptionalRecord(agentrun?.imageBuild); + const gitopsPublish = asOptionalRecord(agentrun?.gitopsPublish); + for (const stage of [ + k8sJobTiming("git-mirror-sync", asOptionalRecord(agentrunSync?.payload), stringOrNull(agentrunSync?.jobName)), + k8sJobTiming("image-build", asOptionalRecord(imageBuild?.result), stringOrNull(imageBuild?.jobName)), + k8sJobTiming("gitops-publish", asOptionalRecord(gitopsPublish?.result), stringOrNull(gitopsPublish?.jobName)), + k8sJobTiming("git-mirror-flush", asOptionalRecord(agentrunFlush?.payload), stringOrNull(agentrunFlush?.jobName)), + ]) if (stage !== null) stages.push(stage); + const tektonSeconds = secondsFromMs(numberOrNull(payload.elapsedMs)); + if (tektonSeconds !== null) { + const status = payload.completed === true ? "completed" : payload.failed === true ? "failed" : payload.stillRunning === true ? "running" : "submitted"; + stages.push(stageTiming("pipelinerun-wait", status, tektonSeconds, null, "tekton-submit", stringOrNull(command.pipelineRun))); + } + } + const closeout = asOptionalRecord(command.closeout); + if (closeout !== null) { + const gitMirrorFlush = asOptionalRecord(closeout.gitMirrorFlush); + const gitMirrorFlushStage = k8sJobTiming("git-mirror-flush", asOptionalRecord(gitMirrorFlush?.result), stringOrNull(gitMirrorFlush?.jobName)); + if (gitMirrorFlushStage !== null) stages.push(gitMirrorFlushStage); + const status = closeout.completed === true ? "completed" : closeout.timedOut === true ? "over-budget" : "pending"; + stages.push(stageTiming("closeout", status, secondsFromMs(numberOrNull(closeout.elapsedMs)), null, "k8s-native-closeout", stringOrNull(command.pipelineRun))); + } + return stages; +} + +function k8sJobTiming(stage: string, job: Record | null, objectOverride?: string | null): StageTiming | null { + if (job === null) return null; + const status = job.completed === true ? job.reused === true ? "reused" : "completed" : job.failed === true ? "failed" : job.timedOut === true ? "over-budget" : "running"; + return stageTiming(stage, status, secondsFromMs(numberOrNull(job.elapsedMs)), null, "kubernetes-job", objectOverride ?? stringOrNull(job.jobName)); +} + +function stageTiming(stage: string, status: string, seconds: number | null, budgetSeconds: number | null, source: string, object: string | null): StageTiming { + return { stage, status, seconds, budgetSeconds, source, object }; +} + +function dedupeTimingStages(stages: StageTiming[]): StageTiming[] { + const byKey = new Map(); + for (const stage of stages) { + if (stage.stage.length === 0) continue; + const key = `${stage.stage}\t${stage.object ?? ""}`; + const previous = byKey.get(key); + if (previous === undefined || previous.seconds === null && stage.seconds !== null) byKey.set(key, stage); + } + return [...byKey.values()]; +} + +function secondsFromMs(value: number | null): number | null { + return value === null ? null : roundSeconds(value / 1000); +} + +function roundSeconds(value: number): number { + return Math.round(value * 10) / 10; +} + +function timestampMs(value: string | null): number | null { + if (value === null) return null; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function asOptionalRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; +} + +function arrayRecords(value: unknown): Record[] { + return Array.isArray(value) ? value.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; +} + +function stringOrNull(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +}