mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1535 from pikasTech/fix/1534-perf-visibility

Browse Source 
Fix branch follower performance status visibility
This commit is contained in:
Lyon
2026-07-04 21:08:50 +08:00
committed by GitHub
parent 3d52db184c c3e97e2745
commit 0fde347fae
4 changed files with 388 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/cicd-branch-follower.ts
+5 -313
View File
@@ -30,14 +30,14 @@ import { compactRefreshEvidence, followerEvidenceSummary } from "./cicd-evidence
import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native"; import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native";
import { argoApplicationReady, nativeArgoSummary, nativeGitMirrorReady, nativeGitMirrorRequired, nativeGitMirrorSummary, nativePipelineRunSummary, nativeRuntimeSummary, pipelineRunSucceeded, runtimeTargetShaFromWorkloads, runtimeWorkloadsReady } from "./cicd-native-summary"; import { argoApplicationReady, nativeArgoSummary, nativeGitMirrorReady, nativeGitMirrorRequired, nativeGitMirrorSummary, nativePipelineRunSummary, nativeRuntimeSummary, pipelineRunSucceeded, runtimeTargetShaFromWorkloads, runtimeWorkloadsReady } from "./cicd-native-summary";
import { invalidRuntimeReuseConfig, missingRuntimeReuseConfig, parseRuntimeReuseConfig, requiredReuseServiceError, RUNTIME_REUSE_CONFIG_PATH, summarizeRuntimeReuseConfig, type RuntimeReuseConfig } from "./cicd-reuse-config"; import { invalidRuntimeReuseConfig, missingRuntimeReuseConfig, parseRuntimeReuseConfig, requiredReuseServiceError, RUNTIME_REUSE_CONFIG_PATH, summarizeRuntimeReuseConfig, type RuntimeReuseConfig } from "./cicd-reuse-config";
import { prioritizedTaskRunItems } from "./cicd-taskruns";
import { runBranchFollowerTaskRunDrillDown } from "./cicd-taskrun-drilldown"; import { runBranchFollowerTaskRunDrillDown } from "./cicd-taskrun-drilldown";
import { runBranchFollowerJobDrillDown, runBranchFollowerRuntimeDrillDown } from "./cicd-job-runtime-drilldown"; import { runBranchFollowerJobDrillDown, runBranchFollowerRuntimeDrillDown } from "./cicd-job-runtime-drilldown";
import { runBranchFollowerGate } from "./cicd-gates"; import { runBranchFollowerGate } from "./cicd-gates";
import { buildCicdHelp } from "./cicd-help"; import { buildCicdHelp } from "./cicd-help";
import { attachReconcileTimeline, compactReconcileTimeline, finishReconcileStep, finishReconcileTimeline, startReconcileStep, startReconcileTimeline } from "./cicd-reconcile-timeline"; import { attachReconcileTimeline, compactReconcileTimeline, finishReconcileStep, finishReconcileTimeline, startReconcileStep, startReconcileTimeline } from "./cicd-reconcile-timeline";
import { orderFollowersForControllerCloseout, shouldYieldAfterAutomaticTrigger } from "./cicd-reconcile-scheduler"; import { orderFollowersForControllerCloseout, shouldYieldAfterAutomaticTrigger } from "./cicd-reconcile-scheduler";
import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerGate, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types"; import { buildFollowerTimings, compactListTimings, compactTimings, storedFollowerTimingsForStatus, timingPerformanceSummary } from "./cicd-timings";
import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerGate, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, TriggerResult } from "./cicd-types";
import { import {
arrayField, arrayField,
asRecord, asRecord,
@@ -633,7 +633,7 @@ async function buildStatus(registry: BranchFollowerRegistry, options: ParsedOpti
const shouldLive = wantsLive && options.inCluster; const shouldLive = wantsLive && options.inCluster;
const selected = selectFollowers(registry, options, { includeDisabled: true }); const selected = selectFollowers(registry, options, { includeDisabled: true });
const followers = []; const followers = [];
const detailedFollowers = options.followerId !== null || options.full; const detailedFollowers = options.full;
for (const follower of selected) { for (const follower of selected) {
const stored = k8s.stateByFollower[follower.id] ?? {}; const stored = k8s.stateByFollower[follower.id] ?? {};
const fallbackStored = refresh === null ? {} : beforeRefreshStateByFollower[follower.id] ?? {}; const fallbackStored = refresh === null ? {} : beforeRefreshStateByFollower[follower.id] ?? {};
@@ -1995,10 +1995,12 @@ function mergeFollowerStatus(
lastSucceededSha, lastSucceededSha,
pipelineRun: live?.pipelineRun ?? stringOrNull(stored.pipelineRun), pipelineRun: live?.pipelineRun ?? stringOrNull(stored.pipelineRun),
inFlightJob: live?.inFlightJob ?? stringOrNull(stored.inFlightJob), inFlightJob: live?.inFlightJob ?? stringOrNull(stored.inFlightJob),
budgetSource: follower.budgets,
updatedAt: stringOrNull(stored.updatedAt), updatedAt: stringOrNull(stored.updatedAt),
live: liveRequested, live: liveRequested,
message: live?.message ?? stringOrNull(stored.decision) ?? "no controller state yet", message: live?.message ?? stringOrNull(stored.decision) ?? "no controller state yet",
timings: detailed ? timings : compactListTimings(timings), timings: detailed ? timings : compactListTimings(timings),
performance: timingPerformanceSummary(timings),
evidence: detailed ? evidence : null, evidence: detailed ? evidence : null,
reconcileTimeline: detailed ? reconcileTimeline : null, reconcileTimeline: detailed ? reconcileTimeline : null,
rawStateDiagnostic: detailed ? asOptionalRecord(stored.rawStateDiagnostic) : null, rawStateDiagnostic: detailed ? asOptionalRecord(stored.rawStateDiagnostic) : null,
@@ -2011,28 +2013,12 @@ function mergeFollowerStatus(
...asOptionalRecord(summary.source), ...asOptionalRecord(summary.source),
snapshotPrefix: follower.source.snapshotPrefix, snapshotPrefix: follower.source.snapshotPrefix,
}, },
budgetSource: follower.budgets,
stateConfigMap: registry.controller.stateConfigMapName, stateConfigMap: registry.controller.stateConfigMapName,
warnings: Array.isArray(stored.warnings) ? stored.warnings.slice(0, 6) : [], warnings: Array.isArray(stored.warnings) ? stored.warnings.slice(0, 6) : [],
next: followerNextCommands(follower), next: followerNextCommands(follower),
}; };
} }
function compactListTimings(timings: FollowerState["timings"]): Record<string, unknown> {
return {
budgetSeconds: timings.budgetSeconds,
totalSeconds: timings.totalSeconds,
totalStatus: timings.totalStatus,
sourceCommit: timings.sourceCommit,
overBudget: timings.overBudget,
stages: timings.stages.slice(0, 4).map((stage) => ({
stage: stage.stage,
status: stage.status,
seconds: stage.seconds,
})),
};
}
function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead { function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead {
const errors: string[] = []; const errors: string[] = [];
const stateResult = kubeConfigMapFollowerState(registry, options); const stateResult = kubeConfigMapFollowerState(registry, options);
@@ -2405,300 +2391,6 @@ function compactSourcePayload(source: Record<string, unknown> | null): Record<st
}; };
} }
function buildFollowerTimings(
follower: FollowerSpec,
live: AdapterSummary,
triggerCommand: Record<string, unknown> | undefined,
storedTimings?: Record<string, unknown> | null,
phase?: BranchFollowerPhase,
): FollowerState["timings"] {
const nativePayload = asOptionalRecord(live.payload);
const finishOverride = stringOrNull(triggerCommand?.finishedAt) ?? noopStoredTotalFinishOverride(storedTimings, phase, live);
const total = totalTimingFromCommand(triggerCommand, phase) ?? totalTimingFromStored(storedTimings, phase, finishOverride, live.observedSha);
const storedStages = live.observedSha !== null && stringOrNull(storedTimings?.sourceCommit) === live.observedSha ? storedStageTimings(storedTimings ?? null) : [];
const stages = dedupeTimingStages([
...stageTimingsFromCommand(triggerCommand),
...stageTimingsFromNativePayload(nativePayload),
...storedStages,
]).slice(0, 24);
const stageSourceCommit = stages.length > 0 ? live.observedSha : null;
return {
budgetSeconds: follower.budgets.endToEndSeconds,
totalSeconds: total?.seconds ?? null,
totalStatus: total?.status ?? "unknown",
totalSource: total?.source ?? "-",
sourceCommit: total?.sourceCommit ?? stringOrNull(triggerCommand?.sourceCommit) ?? stageSourceCommit,
startedAt: total?.startedAt ?? null,
finishedAt: total?.finishedAt ?? null,
overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds,
stages,
};
}
function noopStoredTotalFinishOverride(
storedTimings: Record<string, unknown> | null | undefined,
phase: BranchFollowerPhase | undefined,
live: AdapterSummary,
): string | null {
if (phase !== "Noop" || live.aligned !== true || live.observedSha === null) return null;
if (storedTimings === null || storedTimings === undefined) return null;
if (stringOrNull(storedTimings.sourceCommit) !== live.observedSha) return null;
if (stringOrNull(storedTimings.startedAt) === null) return null;
if (stringOrNull(storedTimings.finishedAt) !== null) return null;
return new Date().toISOString();
}
function storedFollowerTimingsForStatus(
follower: FollowerSpec,
storedTimings: Record<string, unknown> | null,
phase: BranchFollowerPhase,
observedSha: string | null,
): FollowerState["timings"] {
const total = totalTimingFromStored(storedTimings, phase, null, observedSha);
const sourceCommit = total?.sourceCommit ?? stringOrNull(storedTimings?.sourceCommit) ?? null;
return {
budgetSeconds: follower.budgets.endToEndSeconds,
totalSeconds: total?.seconds ?? null,
totalStatus: total?.status ?? "unknown",
totalSource: total?.source ?? "-",
sourceCommit,
startedAt: total?.startedAt ?? null,
finishedAt: total?.finishedAt ?? null,
overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds,
stages: sourceCommit === null ? [] : storedStageTimings(storedTimings),
};
}
function storedStageTimings(storedTimings: Record<string, unknown> | null): StageTiming[] {
if (storedTimings === null) return [];
return arrayRecords(storedTimings.stages)
.map((stage) => stageTiming(
stringOrNull(stage.stage) ?? "",
stringOrNull(stage.status) ?? "unknown",
numberOrNull(stage.seconds),
numberOrNull(stage.budgetSeconds),
stringOrNull(stage.source) ?? "stored-state",
stringOrNull(stage.object),
))
.filter((stage) => stage.stage.length > 0)
.slice(0, 24);
}
function compactTimings(timings: FollowerState["timings"]): FollowerState["timings"] {
return {
budgetSeconds: timings.budgetSeconds,
totalSeconds: timings.totalSeconds,
totalStatus: timings.totalStatus,
totalSource: timings.totalSource,
sourceCommit: timings.sourceCommit,
startedAt: timings.startedAt,
finishedAt: timings.finishedAt,
overBudget: timings.overBudget,
stages: timings.stages.slice(0, 24).map((stage) => ({
stage: stage.stage,
status: stage.status,
seconds: stage.seconds,
budgetSeconds: stage.budgetSeconds,
source: stage.source,
object: stage.object,
})),
};
}
function totalTimingFromCommand(command: Record<string, unknown> | undefined, phase?: BranchFollowerPhase): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null {
if (command === undefined) return null;
if (command.mode === "k8s-native-closeout") return null;
const payload = asOptionalRecord(command.payload);
if (payload?.reused === true) return null;
const startedAt = stringOrNull(command.startedAt);
const finishedAt = stringOrNull(command.finishedAt);
const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? secondsFromMs(numberOrNull(command.elapsedMs));
if (seconds === null) return null;
const closeout = asOptionalRecord(command.closeout);
const exitCode = numberOrNull(command.exitCode);
const status = command.ok === false || (exitCode !== null && exitCode !== 0)
? "failed"
: command.budgetTimedOut === true || closeout?.timedOut === true
? "over-budget"
: command.timedOut === true
? "timed-out"
: closeout?.completed === true || command.completed === true
? "completed"
: command.stillRunning === true
? "running"
: command.pipelineRunCompleted === true
? "ci-completed"
: phase === undefined
? "submitted"
: phase.toLowerCase();
return { seconds, status, source: stringOrNull(command.mode) ?? stringOrNull(command.status) ?? "command", sourceCommit: stringOrNull(command.sourceCommit), startedAt, finishedAt };
}
function totalTimingFromStored(storedTimings: Record<string, unknown> | null | undefined, phase?: BranchFollowerPhase, finishOverride?: string | null, observedSha?: string | null): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null {
if (storedTimings === null || storedTimings === undefined) return null;
const status = stringOrNull(storedTimings.totalStatus);
const source = stringOrNull(storedTimings.totalSource);
const sourceCommit = stringOrNull(storedTimings.sourceCommit);
if (sourceCommit === null) return null;
if (observedSha !== null && observedSha !== undefined && sourceCommit !== observedSha) return null;
const startedAt = stringOrNull(storedTimings.startedAt);
const finishedAt = stringOrNull(storedTimings.finishedAt) ?? finishOverride ?? null;
if (phase === "Noop" && finishedAt === null) return null;
const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? numberOrNull(storedTimings.totalSeconds);
if (seconds === null) return null;
return {
seconds,
status: finishedAt === null && phase !== undefined && !terminalPhase(phase) ? phase.toLowerCase() : phase === undefined ? status ?? "recorded" : phase.toLowerCase(),
source: source ?? "stored-state",
sourceCommit,
startedAt,
finishedAt,
};
}
function totalSecondsFromRange(startedAt: string | null, finishedAt: string | null): number | null {
const startedMs = timestampMs(startedAt);
if (startedMs === null) return null;
const finishedMs = timestampMs(finishedAt) ?? Date.now();
return finishedMs >= startedMs ? roundSeconds((finishedMs - startedMs) / 1000) : null;
}
function timestampMs(value: string | null): number | null {
if (value === null) return null;
const parsed = Date.parse(value);
return Number.isFinite(parsed) ? parsed : null;
}
function terminalPhase(phase: BranchFollowerPhase): boolean {
return phase === "Succeeded" || phase === "Failed" || phase === "Blocked" || phase === "Skipped" || phase === "Noop";
}
function stageTimingsFromNativePayload(payload: Record<string, unknown> | null): StageTiming[] {
if (payload === null) return [];
const stages: StageTiming[] = [];
const statusRead = asOptionalRecord(asOptionalRecord(payload.timings)?.statusRead);
stages.push(stageTiming("status-read", "ok", secondsFromMs(numberOrNull(statusRead?.elapsedMs)), numberOrNull(statusRead?.budgetSeconds), "native-status", null));
const sourceSyncStage = k8sJobTiming("git-mirror-sync", asOptionalRecord(payload.sourceSync));
if (sourceSyncStage !== null) stages.push(sourceSyncStage);
const reuseConfig = asOptionalRecord(payload.reuseConfig);
if (reuseConfig !== null) {
stages.push(stageTiming("reuse-config", reuseConfig.ok === true ? "ready" : "missing-or-invalid", null, null, "source-gitops", stringOrNull(reuseConfig.path)));
}
const gitMirror = asOptionalRecord(payload.gitMirror);
if (gitMirror !== null) {
const hasGitopsBranch = stringOrNull(gitMirror.gitopsBranch) !== null;
const sourceReady = gitMirror.sourceSnapshotReady === true;
const status = gitMirror.pendingFlush === true
? "pending-flush"
: hasGitopsBranch
? gitMirror.githubInSync === true && sourceReady ? "ready" : "not-ready"
: sourceReady ? "source-ready" : "source-not-ready";
stages.push(stageTiming("git-mirror", status, null, null, "git-mirror-cache", stringOrNull(gitMirror.gitopsBranch) ?? stringOrNull(gitMirror.sourceBranch)));
}
const tekton = asOptionalRecord(payload.tekton);
if (tekton !== null) {
const status = tekton.succeeded === true ? "succeeded" : tekton.succeeded === false ? `failed:${stringOrNull(tekton.reason) ?? "unknown"}` : "running";
stages.push(stageTiming("pipelinerun", status, numberOrNull(tekton.durationSeconds), null, "tekton", stringOrNull(tekton.name)));
}
const taskRuns = asOptionalRecord(payload.taskRuns);
for (const record of taskRuns === null ? [] : prioritizedTaskRunItems(taskRuns)) {
const name = stringOrNull(record.pipelineTask) ?? stringOrNull(record.name) ?? "unknown";
const status = record.status === "True" ? "succeeded" : record.status === "False" ? `failed:${stringOrNull(record.reason) ?? "unknown"}` : "running";
stages.push(stageTiming(`task:${name}`, status, numberOrNull(record.durationSeconds), null, "tekton-taskrun", stringOrNull(record.name)));
}
const argo = asOptionalRecord(payload.argo);
if (argo !== null) {
stages.push(stageTiming("argo", `${stringOrNull(argo.syncStatus) ?? "unknown"}/${stringOrNull(argo.healthStatus) ?? "unknown"}`, numberOrNull(argo.operationDurationSeconds), null, "argocd", stringOrNull(argo.name)));
}
const runtime = asOptionalRecord(payload.runtime);
if (runtime !== null) {
const aligned = runtime.aligned === true ? "aligned" : runtime.aligned === false ? "stale" : "unknown-target";
stages.push(stageTiming("runtime", `${runtime.ready === true ? "ready" : "not-ready"}/${aligned}`, null, null, "kubernetes-workload", stringOrNull(runtime.namespace)));
}
return stages;
}
function stageTimingsFromCommand(command: Record<string, unknown> | undefined): StageTiming[] {
if (command === undefined) return [];
const stages: StageTiming[] = [];
const phase = stringOrNull(command.phase);
const jobStage = phase === null ? null : k8sJobTiming(phase, asOptionalRecord(command.job), stringOrNull(command.jobName));
if (jobStage !== null) stages.push(jobStage);
const payload = asOptionalRecord(command.payload);
if (payload !== null) {
const capabilities = asOptionalRecord(payload.nativeCapabilities);
for (const stage of [
k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)),
k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)),
k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)),
]) {
if (stage !== null) stages.push(stage);
}
const agentrun = asOptionalRecord(payload.agentrun);
const agentrunSync = asOptionalRecord(agentrun?.gitMirrorSync);
const agentrunFlush = asOptionalRecord(agentrun?.gitMirrorFlush);
const imageBuild = asOptionalRecord(agentrun?.imageBuild);
const gitopsPublish = asOptionalRecord(agentrun?.gitopsPublish);
for (const stage of [
k8sJobTiming("git-mirror-sync", asOptionalRecord(agentrunSync?.payload), stringOrNull(agentrunSync?.jobName)),
k8sJobTiming("image-build", asOptionalRecord(imageBuild?.result), stringOrNull(imageBuild?.jobName)),
k8sJobTiming("gitops-publish", asOptionalRecord(gitopsPublish?.result), stringOrNull(gitopsPublish?.jobName)),
k8sJobTiming("git-mirror-flush", asOptionalRecord(agentrunFlush?.payload), stringOrNull(agentrunFlush?.jobName)),
]) {
if (stage !== null) stages.push(stage);
}
const tektonSeconds = secondsFromMs(numberOrNull(payload.elapsedMs));
if (tektonSeconds !== null) {
const status = payload.completed === true ? "completed" : payload.failed === true ? "failed" : payload.stillRunning === true ? "running" : "submitted";
stages.push(stageTiming("pipelinerun-wait", status, tektonSeconds, null, "tekton-submit", stringOrNull(command.pipelineRun)));
}
}
const closeout = asOptionalRecord(command.closeout);
if (closeout !== null) {
const gitMirrorFlush = asOptionalRecord(closeout.gitMirrorFlush);
const gitMirrorFlushStage = k8sJobTiming("git-mirror-flush", asOptionalRecord(gitMirrorFlush?.result), stringOrNull(gitMirrorFlush?.jobName));
if (gitMirrorFlushStage !== null) stages.push(gitMirrorFlushStage);
const status = closeout.completed === true ? "completed" : closeout.timedOut === true ? "over-budget" : "pending";
stages.push(stageTiming("closeout", status, secondsFromMs(numberOrNull(closeout.elapsedMs)), null, "k8s-native-closeout", stringOrNull(command.pipelineRun)));
}
return stages;
}
function k8sJobTiming(stage: string, job: Record<string, unknown> | null, objectOverride?: string | null): StageTiming | null {
if (job === null) return null;
const status = job.completed === true
? job.reused === true ? "reused" : "completed"
: job.failed === true
? "failed"
: job.timedOut === true
? "over-budget"
: "running";
return stageTiming(stage, status, secondsFromMs(numberOrNull(job.elapsedMs)), null, "kubernetes-job", objectOverride ?? stringOrNull(job.jobName));
}
function stageTiming(stage: string, status: string, seconds: number | null, budgetSeconds: number | null, source: string, object: string | null): StageTiming {
return { stage, status, seconds, budgetSeconds, source, object };
}
function dedupeTimingStages(stages: StageTiming[]): StageTiming[] {
const byKey = new Map<string, StageTiming>();
for (const stage of stages) {
if (stage.stage.length === 0) continue;
const key = `${stage.stage}\t${stage.object ?? ""}`;
const previous = byKey.get(key);
if (previous === undefined || previous.seconds === null && stage.seconds !== null) byKey.set(key, stage);
}
return [...byKey.values()];
}
function secondsFromMs(value: number | null): number | null {
return value === null ? null : roundSeconds(value / 1000);
}
function roundSeconds(value: number): number {
return Math.round(value * 10) / 10;
}
function writeFollowerState(registry: BranchFollowerRegistry, state: FollowerState, options: ParsedOptions): CommandResult { function writeFollowerState(registry: BranchFollowerRegistry, state: FollowerState, options: ParsedOptions): CommandResult {
const stateJson = JSON.stringify(compactFollowerStateForConfigMap(state)); const stateJson = JSON.stringify(compactFollowerStateForConfigMap(state));
const script = [ const script = [
scripts/src/cicd-evidence.ts
+13 -1
View File
@@ -43,7 +43,7 @@ export function followerEvidenceSummary(input: {
const refreshSourceCommit = stringOrNull(refresh?.sourceCommit); const refreshSourceCommit = stringOrNull(refresh?.sourceCommit);
return { return {
pipelineRunRefName: pipelineRefName, pipelineRunRefName: pipelineRefName,
pipeline, pipeline: compactPipelineEvidence(pipeline),
refreshBoundedReason: refresh === null ? "missing-from-live-and-stored-evidence" : null, refreshBoundedReason: refresh === null ? "missing-from-live-and-stored-evidence" : null,
refresh: refresh === null refresh: refresh === null
? null ? null
@@ -57,6 +57,18 @@ export function followerEvidenceSummary(input: {
}; };
} }
function compactPipelineEvidence(value: Record<string, unknown> | null): Record<string, unknown> | null {
if (value === null) return null;
const metadata = asOptionalRecord(value.metadata);
const spec = asOptionalRecord(value.spec);
return {
metadata: { name: stringOrNull(metadata?.name) },
spec: {
runtimeReadyTask: compactRefreshRuntimeReady(asOptionalRecord(spec?.runtimeReadyTask)),
},
};
}
function compactRefreshRender(value: Record<string, unknown> | null): Record<string, unknown> | null { function compactRefreshRender(value: Record<string, unknown> | null): Record<string, unknown> | null {
if (value === null) return null; if (value === null) return null;
return { return {
scripts/src/cicd-render.ts
+15
View File
@@ -110,6 +110,7 @@ function renderStatusHuman(payload: Record<string, unknown>, _options: ParsedOpt
const next = asOptionalRecord(payload.next); const next = asOptionalRecord(payload.next);
const errors = Array.isArray(payload.errors) ? payload.errors : []; const errors = Array.isArray(payload.errors) ? payload.errors : [];
const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48); const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48);
const performanceRows = followers.flatMap(performanceRowsForFollower).slice(0, 24);
const evidenceRows = followers.flatMap(evidenceRowsForFollower).slice(0, 48); const evidenceRows = followers.flatMap(evidenceRowsForFollower).slice(0, 48);
const reconcileRows = followers.flatMap(reconcileRowsForFollower).slice(0, 48); const reconcileRows = followers.flatMap(reconcileRowsForFollower).slice(0, 48);
const rawStateRows = followers.flatMap(rawStateRowsForFollower).slice(0, 24); const rawStateRows = followers.flatMap(rawStateRowsForFollower).slice(0, 24);
@@ -123,6 +124,7 @@ function renderStatusHuman(payload: Record<string, unknown>, _options: ParsedOpt
"", "",
table(["FOLLOWER", "PHASE", "ADAPTER", "OBSERVED", "TARGET", "TRIGGERED", "SUCCEEDED", "IN_FLIGHT", "BUDGET", "MESSAGE"], rows), table(["FOLLOWER", "PHASE", "ADAPTER", "OBSERVED", "TARGET", "TRIGGERED", "SUCCEEDED", "IN_FLIGHT", "BUDGET", "MESSAGE"], rows),
timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`, timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`,
performanceRows.length === 0 ? "" : `\nSLOW STAGES\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "SOURCE", "OBJECT"], performanceRows)}`,
evidenceRows.length === 0 ? "" : `\nEVIDENCE\n${table(["FOLLOWER", "TYPE", "STATUS", "DETAIL", "OBJECT"], evidenceRows)}`, evidenceRows.length === 0 ? "" : `\nEVIDENCE\n${table(["FOLLOWER", "TYPE", "STATUS", "DETAIL", "OBJECT"], evidenceRows)}`,
reconcileRows.length === 0 ? "" : `\nRECONCILE TIMELINE\n${table(["FOLLOWER", "STEP", "STATUS", "SECONDS", "STARTED", "OBJECT"], reconcileRows)}`, reconcileRows.length === 0 ? "" : `\nRECONCILE TIMELINE\n${table(["FOLLOWER", "STEP", "STATUS", "SECONDS", "STARTED", "OBJECT"], reconcileRows)}`,
rawStateRows.length === 0 ? "" : `\nRAW STATE DIAGNOSTIC\n${table(["FOLLOWER", "STATE_BYTES", "COMMAND", "TIMELINE", "STEPS", "TIMELINE_BYTES", "REASON"], rawStateRows)}`, rawStateRows.length === 0 ? "" : `\nRAW STATE DIAGNOSTIC\n${table(["FOLLOWER", "STATE_BYTES", "COMMAND", "TIMELINE", "STEPS", "TIMELINE_BYTES", "REASON"], rawStateRows)}`,
@@ -231,6 +233,19 @@ function timingRowsForFollower(item: Record<string, unknown>): unknown[][] {
return rows; return rows;
} }
function performanceRowsForFollower(item: Record<string, unknown>): unknown[][] {
const performance = asOptionalRecord(item.performance);
if (performance === null) return [];
return arrayRecords(performance.slowStages).map((stage) => [
item.id,
stage.stage ?? "-",
stage.status ?? "-",
formatSeconds(numberOrNull(stage.seconds)),
stringOrNull(stage.source) ?? "-",
stringOrNull(stage.object) ?? "-",
]);
}
function reconcileRowsFromRunOnce(payload: Record<string, unknown>, followers: Record<string, unknown>[]): unknown[][] { function reconcileRowsFromRunOnce(payload: Record<string, unknown>, followers: Record<string, unknown>[]): unknown[][] {
const timeline = asOptionalRecord(payload.reconcileTimeline); const timeline = asOptionalRecord(payload.reconcileTimeline);
if (timeline !== null) return reconcileRowsForTimeline(timeline, null); if (timeline !== null) return reconcileRowsForTimeline(timeline, null);
scripts/src/cicd-timings.ts
+355
View File
@@ -0,0 +1,355 @@
// SPEC: PJ2026-01060703 CI/CD branch follower timing helpers.
// Responsibility: compact branch-follower total/stage timing contracts and performance summaries.
import { prioritizedTaskRunItems } from "./cicd-taskruns";
import type { AdapterSummary, BranchFollowerPhase, FollowerSpec, FollowerState, StageTiming } from "./cicd-types";
type TotalTiming = {
seconds: number;
status: string;
source: string;
sourceCommit: string | null;
startedAt: string | null;
finishedAt: string | null;
};
export function buildFollowerTimings(
follower: FollowerSpec,
live: AdapterSummary,
triggerCommand: Record<string, unknown> | undefined,
storedTimings?: Record<string, unknown> | null,
phase?: BranchFollowerPhase,
): FollowerState["timings"] {
const nativePayload = asOptionalRecord(live.payload);
const finishOverride = stringOrNull(triggerCommand?.finishedAt) ?? noopStoredTotalFinishOverride(storedTimings, phase, live);
const total = totalTimingFromCommand(triggerCommand, phase) ?? totalTimingFromStored(storedTimings, phase, finishOverride, live.observedSha);
const storedStages = live.observedSha !== null && stringOrNull(storedTimings?.sourceCommit) === live.observedSha ? storedStageTimings(storedTimings ?? null) : [];
const stages = dedupeTimingStages([
...stageTimingsFromCommand(triggerCommand),
...stageTimingsFromNativePayload(nativePayload, total),
...storedStages,
]).slice(0, 24);
const stageSourceCommit = stages.length > 0 ? live.observedSha : null;
return {
budgetSeconds: follower.budgets.endToEndSeconds,
totalSeconds: total?.seconds ?? null,
totalStatus: total?.status ?? "unknown",
totalSource: total?.source ?? "-",
sourceCommit: total?.sourceCommit ?? stringOrNull(triggerCommand?.sourceCommit) ?? stageSourceCommit,
startedAt: total?.startedAt ?? null,
finishedAt: total?.finishedAt ?? null,
overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds,
stages,
};
}
export function storedFollowerTimingsForStatus(
follower: FollowerSpec,
storedTimings: Record<string, unknown> | null,
phase: BranchFollowerPhase,
observedSha: string | null,
): FollowerState["timings"] {
const total = totalTimingFromStored(storedTimings, phase, null, observedSha);
const sourceCommit = total?.sourceCommit ?? stringOrNull(storedTimings?.sourceCommit) ?? null;
return {
budgetSeconds: follower.budgets.endToEndSeconds,
totalSeconds: total?.seconds ?? null,
totalStatus: total?.status ?? "unknown",
totalSource: total?.source ?? "-",
sourceCommit,
startedAt: total?.startedAt ?? null,
finishedAt: total?.finishedAt ?? null,
overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds,
stages: sourceCommit === null ? [] : storedStageTimings(storedTimings),
};
}
export function compactTimings(timings: FollowerState["timings"]): FollowerState["timings"] {
return {
budgetSeconds: timings.budgetSeconds,
totalSeconds: timings.totalSeconds,
totalStatus: timings.totalStatus,
totalSource: timings.totalSource,
sourceCommit: timings.sourceCommit,
startedAt: timings.startedAt,
finishedAt: timings.finishedAt,
overBudget: timings.overBudget,
stages: timings.stages.slice(0, 24).map((stage) => ({
stage: stage.stage,
status: stage.status,
seconds: stage.seconds,
budgetSeconds: stage.budgetSeconds,
source: stage.source,
object: stage.object,
})),
};
}
export function compactListTimings(timings: FollowerState["timings"]): Record<string, unknown> {
return {
budgetSeconds: timings.budgetSeconds,
totalSeconds: timings.totalSeconds,
totalStatus: timings.totalStatus,
sourceCommit: timings.sourceCommit,
overBudget: timings.overBudget,
stages: timings.stages.slice(0, 4).map((stage) => ({
stage: stage.stage,
status: stage.status,
seconds: stage.seconds,
})),
};
}
export function timingPerformanceSummary(timings: FollowerState["timings"]): Record<string, unknown> {
const slowStages = timings.stages
.filter((stage) => stage.seconds !== null)
.sort((a, b) => (b.seconds ?? 0) - (a.seconds ?? 0))
.slice(0, 5)
.map((stage) => ({
stage: stage.stage,
status: stage.status,
seconds: stage.seconds,
source: stage.source,
object: stage.object,
}));
return {
budgetSeconds: timings.budgetSeconds,
totalSeconds: timings.totalSeconds,
overBudget: timings.overBudget,
slowestStage: slowStages[0] ?? null,
slowStages,
note: timings.overBudget === true ? "advisory budget exceeded; inspect slowStages and stage source before rerun" : null,
};
}
function noopStoredTotalFinishOverride(
storedTimings: Record<string, unknown> | null | undefined,
phase: BranchFollowerPhase | undefined,
live: AdapterSummary,
): string | null {
if (phase !== "Noop" || live.aligned !== true || live.observedSha === null) return null;
if (storedTimings === null || storedTimings === undefined) return null;
if (stringOrNull(storedTimings.sourceCommit) !== live.observedSha) return null;
if (stringOrNull(storedTimings.startedAt) === null) return null;
if (stringOrNull(storedTimings.finishedAt) !== null) return null;
return new Date().toISOString();
}
function storedStageTimings(storedTimings: Record<string, unknown> | null): StageTiming[] {
if (storedTimings === null) return [];
return arrayRecords(storedTimings.stages)
.map((stage) => stageTiming(
stringOrNull(stage.stage) ?? "",
stringOrNull(stage.status) ?? "unknown",
numberOrNull(stage.seconds),
numberOrNull(stage.budgetSeconds),
stringOrNull(stage.source) ?? "stored-state",
stringOrNull(stage.object),
))
.filter((stage) => stage.stage.length > 0)
.slice(0, 24);
}
function totalTimingFromCommand(command: Record<string, unknown> | undefined, phase?: BranchFollowerPhase): TotalTiming | null {
if (command === undefined) return null;
if (command.mode === "k8s-native-closeout") return null;
const payload = asOptionalRecord(command.payload);
if (payload?.reused === true) return null;
const startedAt = stringOrNull(command.startedAt);
const finishedAt = stringOrNull(command.finishedAt);
const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? secondsFromMs(numberOrNull(command.elapsedMs));
if (seconds === null) return null;
const closeout = asOptionalRecord(command.closeout);
const exitCode = numberOrNull(command.exitCode);
const status = command.ok === false || (exitCode !== null && exitCode !== 0)
? "failed"
: command.budgetTimedOut === true || closeout?.timedOut === true
? "over-budget"
: command.timedOut === true
? "timed-out"
: closeout?.completed === true || command.completed === true
? "completed"
: command.stillRunning === true
? "running"
: command.pipelineRunCompleted === true
? "ci-completed"
: phase === undefined
? "submitted"
: phase.toLowerCase();
return { seconds, status, source: stringOrNull(command.mode) ?? stringOrNull(command.status) ?? "command", sourceCommit: stringOrNull(command.sourceCommit), startedAt, finishedAt };
}
function totalTimingFromStored(storedTimings: Record<string, unknown> | null | undefined, phase?: BranchFollowerPhase, finishOverride?: string | null, observedSha?: string | null): TotalTiming | null {
if (storedTimings === null || storedTimings === undefined) return null;
const status = stringOrNull(storedTimings.totalStatus);
const source = stringOrNull(storedTimings.totalSource);
const sourceCommit = stringOrNull(storedTimings.sourceCommit);
if (sourceCommit === null) return null;
if (observedSha !== null && observedSha !== undefined && sourceCommit !== observedSha) return null;
const startedAt = stringOrNull(storedTimings.startedAt);
const finishedAt = stringOrNull(storedTimings.finishedAt) ?? finishOverride ?? null;
if (phase === "Noop" && finishedAt === null) return null;
const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? numberOrNull(storedTimings.totalSeconds);
if (seconds === null) return null;
return {
seconds,
status: finishedAt === null && phase !== undefined && !terminalPhase(phase) ? phase.toLowerCase() : phase === undefined ? status ?? "recorded" : phase.toLowerCase(),
source: source ?? "stored-state",
sourceCommit,
startedAt,
finishedAt,
};
}
function totalSecondsFromRange(startedAt: string | null, finishedAt: string | null): number | null {
const startedMs = timestampMs(startedAt);
if (startedMs === null) return null;
const finishedMs = timestampMs(finishedAt) ?? Date.now();
return finishedMs >= startedMs ? roundSeconds((finishedMs - startedMs) / 1000) : null;
}
function terminalPhase(phase: BranchFollowerPhase): boolean {
return phase === "Succeeded" || phase === "Failed" || phase === "Blocked" || phase === "Skipped" || phase === "Noop";
}
function stageTimingsFromNativePayload(payload: Record<string, unknown> | null, total: TotalTiming | null): StageTiming[] {
if (payload === null) return [];
const stages: StageTiming[] = [];
const statusRead = asOptionalRecord(asOptionalRecord(payload.timings)?.statusRead);
stages.push(stageTiming("status-read", "ok", secondsFromMs(numberOrNull(statusRead?.elapsedMs)), numberOrNull(statusRead?.budgetSeconds), "native-status", null));
const sourceSyncStage = k8sJobTiming("git-mirror-sync", asOptionalRecord(payload.sourceSync));
if (sourceSyncStage !== null) stages.push(sourceSyncStage);
const reuseConfig = asOptionalRecord(payload.reuseConfig);
if (reuseConfig !== null) stages.push(stageTiming("reuse-config", reuseConfig.ok === true ? "ready" : "missing-or-invalid", null, null, "source-gitops", stringOrNull(reuseConfig.path)));
const gitMirror = asOptionalRecord(payload.gitMirror);
if (gitMirror !== null) {
const hasGitopsBranch = stringOrNull(gitMirror.gitopsBranch) !== null;
const sourceReady = gitMirror.sourceSnapshotReady === true;
const status = gitMirror.pendingFlush === true ? "pending-flush" : hasGitopsBranch ? gitMirror.githubInSync === true && sourceReady ? "ready" : "not-ready" : sourceReady ? "source-ready" : "source-not-ready";
stages.push(stageTiming("git-mirror", status, null, null, "git-mirror-cache", stringOrNull(gitMirror.gitopsBranch) ?? stringOrNull(gitMirror.sourceBranch)));
}
const tekton = asOptionalRecord(payload.tekton);
if (tekton !== null) {
const status = tekton.succeeded === true ? "succeeded" : tekton.succeeded === false ? `failed:${stringOrNull(tekton.reason) ?? "unknown"}` : "running";
stages.push(stageTiming("pipelinerun", status, numberOrNull(tekton.durationSeconds), null, "tekton", stringOrNull(tekton.name)));
}
const taskRuns = asOptionalRecord(payload.taskRuns);
for (const record of taskRuns === null ? [] : prioritizedTaskRunItems(taskRuns)) {
const name = stringOrNull(record.pipelineTask) ?? stringOrNull(record.name) ?? "unknown";
const status = record.status === "True" ? "succeeded" : record.status === "False" ? `failed:${stringOrNull(record.reason) ?? "unknown"}` : "running";
stages.push(stageTiming(`task:${name}`, status, numberOrNull(record.durationSeconds), null, "tekton-taskrun", stringOrNull(record.name)));
}
const argo = asOptionalRecord(payload.argo);
if (argo !== null) {
const sameWindow = total === null || timingOverlapsTotal(total, stringOrNull(argo.operationStartedAt), stringOrNull(argo.operationFinishedAt));
stages.push(stageTiming("argo", `${sameWindow ? "" : "current:"}${stringOrNull(argo.syncStatus) ?? "unknown"}/${stringOrNull(argo.healthStatus) ?? "unknown"}`, sameWindow ? numberOrNull(argo.operationDurationSeconds) : null, null, sameWindow ? "argocd" : "argocd-current", stringOrNull(argo.name)));
}
const runtime = asOptionalRecord(payload.runtime);
if (runtime !== null) {
const aligned = runtime.aligned === true ? "aligned" : runtime.aligned === false ? "stale" : "unknown-target";
stages.push(stageTiming("runtime", `${runtime.ready === true ? "ready" : "not-ready"}/${aligned}`, null, null, "kubernetes-workload", stringOrNull(runtime.namespace)));
}
return stages;
}
function timingOverlapsTotal(total: TotalTiming, startedAt: string | null, finishedAt: string | null): boolean {
const totalStart = timestampMs(total.startedAt);
const totalFinish = timestampMs(total.finishedAt);
const stageStart = timestampMs(startedAt);
const stageFinish = timestampMs(finishedAt);
if (totalStart === null || totalFinish === null || stageStart === null || stageFinish === null) return true;
const slackMs = 30_000;
return stageStart <= totalFinish + slackMs && stageFinish >= totalStart - slackMs;
}
function stageTimingsFromCommand(command: Record<string, unknown> | undefined): StageTiming[] {
if (command === undefined) return [];
const stages: StageTiming[] = [];
const phase = stringOrNull(command.phase);
const jobStage = phase === null ? null : k8sJobTiming(phase, asOptionalRecord(command.job), stringOrNull(command.jobName));
if (jobStage !== null) stages.push(jobStage);
const payload = asOptionalRecord(command.payload);
if (payload !== null) {
const capabilities = asOptionalRecord(payload.nativeCapabilities);
for (const stage of [
k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)),
k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)),
k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)),
]) if (stage !== null) stages.push(stage);
const agentrun = asOptionalRecord(payload.agentrun);
const agentrunSync = asOptionalRecord(agentrun?.gitMirrorSync);
const agentrunFlush = asOptionalRecord(agentrun?.gitMirrorFlush);
const imageBuild = asOptionalRecord(agentrun?.imageBuild);
const gitopsPublish = asOptionalRecord(agentrun?.gitopsPublish);
for (const stage of [
k8sJobTiming("git-mirror-sync", asOptionalRecord(agentrunSync?.payload), stringOrNull(agentrunSync?.jobName)),
k8sJobTiming("image-build", asOptionalRecord(imageBuild?.result), stringOrNull(imageBuild?.jobName)),
k8sJobTiming("gitops-publish", asOptionalRecord(gitopsPublish?.result), stringOrNull(gitopsPublish?.jobName)),
k8sJobTiming("git-mirror-flush", asOptionalRecord(agentrunFlush?.payload), stringOrNull(agentrunFlush?.jobName)),
]) if (stage !== null) stages.push(stage);
const tektonSeconds = secondsFromMs(numberOrNull(payload.elapsedMs));
if (tektonSeconds !== null) {
const status = payload.completed === true ? "completed" : payload.failed === true ? "failed" : payload.stillRunning === true ? "running" : "submitted";
stages.push(stageTiming("pipelinerun-wait", status, tektonSeconds, null, "tekton-submit", stringOrNull(command.pipelineRun)));
}
}
const closeout = asOptionalRecord(command.closeout);
if (closeout !== null) {
const gitMirrorFlush = asOptionalRecord(closeout.gitMirrorFlush);
const gitMirrorFlushStage = k8sJobTiming("git-mirror-flush", asOptionalRecord(gitMirrorFlush?.result), stringOrNull(gitMirrorFlush?.jobName));
if (gitMirrorFlushStage !== null) stages.push(gitMirrorFlushStage);
const status = closeout.completed === true ? "completed" : closeout.timedOut === true ? "over-budget" : "pending";
stages.push(stageTiming("closeout", status, secondsFromMs(numberOrNull(closeout.elapsedMs)), null, "k8s-native-closeout", stringOrNull(command.pipelineRun)));
}
return stages;
}
function k8sJobTiming(stage: string, job: Record<string, unknown> | null, objectOverride?: string | null): StageTiming | null {
if (job === null) return null;
const status = job.completed === true ? job.reused === true ? "reused" : "completed" : job.failed === true ? "failed" : job.timedOut === true ? "over-budget" : "running";
return stageTiming(stage, status, secondsFromMs(numberOrNull(job.elapsedMs)), null, "kubernetes-job", objectOverride ?? stringOrNull(job.jobName));
}
function stageTiming(stage: string, status: string, seconds: number | null, budgetSeconds: number | null, source: string, object: string | null): StageTiming {
return { stage, status, seconds, budgetSeconds, source, object };
}
function dedupeTimingStages(stages: StageTiming[]): StageTiming[] {
const byKey = new Map<string, StageTiming>();
for (const stage of stages) {
if (stage.stage.length === 0) continue;
const key = `${stage.stage}\t${stage.object ?? ""}`;
const previous = byKey.get(key);
if (previous === undefined || previous.seconds === null && stage.seconds !== null) byKey.set(key, stage);
}
return [...byKey.values()];
}
function secondsFromMs(value: number | null): number | null {
return value === null ? null : roundSeconds(value / 1000);
}
function roundSeconds(value: number): number {
return Math.round(value * 10) / 10;
}
function timestampMs(value: string | null): number | null {
if (value === null) return null;
const parsed = Date.parse(value);
return Number.isFinite(parsed) ? parsed : null;
}
function asOptionalRecord(value: unknown): Record<string, unknown> | null {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
}
function arrayRecords(value: unknown): Record<string, unknown>[] {
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
}
function stringOrNull(value: unknown): string | null {
return typeof value === "string" && value.length > 0 ? value : null;
}
function numberOrNull(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}