Codex
311 lines
14 KiB
TypeScript
311 lines
14 KiB
TypeScript
// SPEC: PJ2026-01060703 CI/CD branch follower debug steps.
|
|
// Responsibility: bounded single-step debugging for branch follower state and decision paths.
|
|
import type { CommandResult } from "./command";
|
|
import type { AdapterSummary, BranchFollowerDebugStep, BranchFollowerRegistry, FollowerSpec, FollowerState, K8sStateRead, ParsedOptions } from "./cicd-types";
|
|
import { renderControllerDebugJob, waitForJobShell } from "./cicd-controller-render";
|
|
import { redactText, shQuote } from "./platform-infra-ops-library";
|
|
|
|
type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult;
|
|
|
|
export interface CicdDebugDeps {
|
|
selectFollowers(registry: BranchFollowerRegistry, options: ParsedOptions, opts: { includeDisabled: boolean }): FollowerSpec[];
|
|
readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead;
|
|
readAdapterStatus(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Promise<AdapterSummary>;
|
|
decideAndMaybeTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, previous: Record<string, unknown>, live: AdapterSummary, options: ParsedOptions): Promise<FollowerState>;
|
|
writeFollowerState(registry: BranchFollowerRegistry, state: FollowerState, options: ParsedOptions): CommandResult;
|
|
runKubeScript: KubeScriptRunner;
|
|
}
|
|
|
|
export async function buildDebugStep(registry: BranchFollowerRegistry, options: ParsedOptions, deps: CicdDebugDeps): Promise<Record<string, unknown>> {
|
|
const step = options.debugStep ?? "state-read";
|
|
if (options.followerId === null) throw new Error("debug-step requires --follower <id>");
|
|
if (!options.inCluster) return runTargetDebugStepJob(registry, options, step, deps);
|
|
|
|
const selected = deps.selectFollowers(registry, options, { includeDisabled: true });
|
|
if (selected.length !== 1) throw new Error("debug-step operates on exactly one follower");
|
|
const follower = selected[0] as FollowerSpec;
|
|
const before = deps.readK8sState(registry, options);
|
|
const previous = before.stateByFollower[follower.id] ?? {};
|
|
let live: AdapterSummary | null = null;
|
|
let decided: FollowerState | null = null;
|
|
let write: Record<string, unknown> | null = null;
|
|
let after: K8sStateRead | null = null;
|
|
|
|
if (step === "status-read" || step === "decide") {
|
|
live = await deps.readAdapterStatus(registry, follower, options);
|
|
}
|
|
if (step === "decide") {
|
|
decided = await deps.decideAndMaybeTrigger(registry, follower, previous, live as AdapterSummary, debugDecisionOptions(options));
|
|
}
|
|
if (step === "state-write") {
|
|
const writeInput = stateWriteInput(previous);
|
|
if (writeInput === null) {
|
|
return {
|
|
ok: false,
|
|
action: "debug-step",
|
|
step,
|
|
follower: follower.id,
|
|
execution: "k8s-native-in-cluster",
|
|
dryRun: !options.confirm,
|
|
stateBefore: stateSnapshot(before, follower.id),
|
|
stateWrite: { ok: false, skipped: true, reason: "stored-state-missing" },
|
|
parsedDownstreamCliOutput: false,
|
|
};
|
|
}
|
|
if (options.confirm) {
|
|
const result = deps.writeFollowerState(registry, writeInput, options);
|
|
write = stateWriteResult(follower.id, result);
|
|
after = deps.readK8sState(registry, options);
|
|
} else {
|
|
write = { ok: true, skipped: true, reason: "dry-run-requires-confirm", input: "stored-state" };
|
|
}
|
|
}
|
|
|
|
return {
|
|
ok: write === null ? before.ok && (live === null || live.ok) : write.ok === true,
|
|
action: "debug-step",
|
|
step,
|
|
follower: follower.id,
|
|
execution: "k8s-native-in-cluster",
|
|
dryRun: !options.confirm,
|
|
stateBefore: stateSnapshot(before, follower.id),
|
|
status: live === null ? null : compactAdapterStatus(live),
|
|
decision: decided === null ? null : compactFollowerDecision(decided),
|
|
stateWrite: write,
|
|
stateAfter: after === null ? null : stateSnapshot(after, follower.id),
|
|
parsedDownstreamCliOutput: false,
|
|
next: debugNext(follower.id),
|
|
};
|
|
}
|
|
|
|
export function renderDebugStepHuman(payload: Record<string, unknown>): string {
|
|
const before = asOptionalRecord(payload.stateBefore);
|
|
const after = asOptionalRecord(payload.stateAfter);
|
|
const write = asOptionalRecord(payload.stateWrite);
|
|
const status = asOptionalRecord(payload.status);
|
|
const decision = asOptionalRecord(payload.decision);
|
|
const target = asOptionalRecord(payload.target);
|
|
const next = asOptionalRecord(payload.next);
|
|
const rows = [[
|
|
payload.follower ?? "-",
|
|
payload.step ?? "-",
|
|
payload.execution ?? "-",
|
|
payload.dryRun === true ? "true" : "false",
|
|
before?.phase ?? "-",
|
|
after?.phase ?? decision?.phase ?? status?.phase ?? "-",
|
|
shortSha(stringOrNull(before?.observedSha)),
|
|
shortSha(stringOrNull(after?.observedSha) ?? stringOrNull(decision?.observedSha) ?? stringOrNull(status?.observedSha)),
|
|
]];
|
|
const writeRows = write === null ? [] : [[
|
|
write.ok === true ? "ok" : "failed",
|
|
write.skipped === true ? "skipped" : "executed",
|
|
write.input ?? "-",
|
|
asOptionalRecord(write.patch)?.beforeResourceVersion ?? "-",
|
|
asOptionalRecord(write.patch)?.afterResourceVersion ?? "-",
|
|
write.exitCode ?? "-",
|
|
write.message ?? write.reason ?? "-",
|
|
]];
|
|
return [
|
|
`CI/CD BRANCH-FOLLOWER DEBUG-STEP (${payload.ok === false ? "failed" : "ok"})`,
|
|
"",
|
|
table(["FOLLOWER", "STEP", "EXECUTION", "DRY_RUN", "BEFORE", "AFTER", "BEFORE_SHA", "AFTER_SHA"], rows),
|
|
target === null ? "" : `\nTARGET JOB\n${table(["JOB", "EXIT", "TIMED_OUT", "PARSED"], [[target.name ?? "-", target.exitCode ?? "-", target.timedOut ?? "-", target.parsed === true ? "yes" : "no"]])}`,
|
|
writeRows.length === 0 ? "" : `\nSTATE WRITE\n${table(["STATUS", "MODE", "INPUT", "BEFORE_RV", "AFTER_RV", "EXIT", "MESSAGE"], writeRows)}`,
|
|
"",
|
|
"NEXT",
|
|
`state-read: ${next?.stateRead ?? "-"}`,
|
|
`status-read: ${next?.statusRead ?? "-"}`,
|
|
`decide: ${next?.decide ?? "-"}`,
|
|
`state-write: ${next?.stateWrite ?? "-"}`,
|
|
"",
|
|
].filter((line) => line !== "").join("\n");
|
|
}
|
|
|
|
function runTargetDebugStepJob(registry: BranchFollowerRegistry, options: ParsedOptions, step: BranchFollowerDebugStep, deps: CicdDebugDeps): Record<string, unknown> {
|
|
const timeoutSeconds = options.timeoutSeconds ?? registry.controller.budgets.runOnceSeconds;
|
|
const jobName = `${registry.controller.deploymentName}-debug-${step}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]+/gu, "-").slice(0, 63);
|
|
const manifest = renderControllerDebugJob(registry, options, jobName, step, timeoutSeconds);
|
|
const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
|
|
const script = [
|
|
"set -eu",
|
|
"tmp=$(mktemp)",
|
|
"base64 -d >\"$tmp\" <<'UNIDESK_CICD_DEBUG_JOB_B64'",
|
|
Buffer.from(manifestYaml, "utf8").toString("base64"),
|
|
"UNIDESK_CICD_DEBUG_JOB_B64",
|
|
`kubectl -n ${shQuote(registry.controller.namespace)} delete job ${shQuote(jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`,
|
|
`kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`,
|
|
waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds),
|
|
].join("\n");
|
|
const result = deps.runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000);
|
|
const parsed = parseLastJsonObject(result.stdout);
|
|
const state = deps.readK8sState(registry, options);
|
|
const followerId = options.followerId ?? "";
|
|
return {
|
|
ok: result.exitCode === 0 && parsed?.ok !== false,
|
|
action: "debug-step",
|
|
step,
|
|
follower: followerId,
|
|
execution: "k8s-native-debug-job",
|
|
dryRun: !options.confirm,
|
|
stateBefore: asOptionalRecord(parsed?.stateBefore),
|
|
status: asOptionalRecord(parsed?.status),
|
|
decision: asOptionalRecord(parsed?.decision),
|
|
stateWrite: asOptionalRecord(parsed?.stateWrite),
|
|
target: {
|
|
name: jobName,
|
|
namespace: registry.controller.namespace,
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
parsed: parsed !== null,
|
|
stdoutTail: redactText(tailText(result.stdout, options.full ? 4000 : 1000)),
|
|
stderrTail: redactText(tailText(result.stderr, options.full ? 2000 : 800)),
|
|
},
|
|
targetResult: parsed,
|
|
stateAfter: asOptionalRecord(parsed?.stateAfter) ?? stateSnapshot(state, followerId),
|
|
parsedDownstreamCliOutput: false,
|
|
next: debugNext(followerId),
|
|
};
|
|
}
|
|
|
|
function debugDecisionOptions(options: ParsedOptions): ParsedOptions {
|
|
return { ...options, confirm: false, dryRun: true, wait: false, recordState: false };
|
|
}
|
|
|
|
function stateWriteInput(previous: Record<string, unknown>): FollowerState | null {
|
|
if (stringOrNull(previous.id) === null) return null;
|
|
if (asOptionalRecord(previous.source) === null || asOptionalRecord(previous.target) === null || asOptionalRecord(previous.timings) === null) return null;
|
|
return previous as unknown as FollowerState;
|
|
}
|
|
|
|
function stateWriteResult(followerId: string, result: CommandResult): Record<string, unknown> {
|
|
const parsed = parseLastJsonObject(result.stdout);
|
|
return {
|
|
ok: result.exitCode === 0 && parsed?.ok !== false,
|
|
follower: followerId,
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
input: "stored-state",
|
|
patch: parsed,
|
|
message: result.exitCode === 0 ? "state patch command completed" : redactText(tailText(result.stderr || result.stdout, 500)),
|
|
parsedDownstreamCliOutput: false,
|
|
};
|
|
}
|
|
|
|
function stateSnapshot(read: K8sStateRead, followerId: string): Record<string, unknown> {
|
|
const state = read.stateByFollower[followerId] ?? {};
|
|
const source = asOptionalRecord(state.source);
|
|
const target = asOptionalRecord(state.target);
|
|
const timings = asOptionalRecord(state.timings);
|
|
return {
|
|
present: read.stateConfigMapPresent,
|
|
ok: read.ok,
|
|
errors: read.errors.slice(0, 8),
|
|
metadata: read.stateMetadata,
|
|
valueBytes: read.stateValueBytes[followerId] ?? null,
|
|
phase: stringOrNull(state.phase),
|
|
observedSha: stringOrNull(source?.observedSha),
|
|
targetSha: stringOrNull(target?.targetSha),
|
|
lastTriggeredSha: stringOrNull(state.lastTriggeredSha),
|
|
lastSucceededSha: stringOrNull(state.lastSucceededSha),
|
|
pipelineRun: stringOrNull(state.pipelineRun),
|
|
inFlightJob: stringOrNull(state.inFlightJob),
|
|
timingStatus: stringOrNull(timings?.totalStatus),
|
|
totalSeconds: numberOrNull(timings?.totalSeconds),
|
|
startedAt: stringOrNull(timings?.startedAt),
|
|
updatedAt: stringOrNull(state.updatedAt),
|
|
};
|
|
}
|
|
|
|
function compactAdapterStatus(live: AdapterSummary): Record<string, unknown> {
|
|
return {
|
|
ok: live.ok,
|
|
phase: live.phase,
|
|
observedSha: live.observedSha,
|
|
targetSha: live.targetSha,
|
|
aligned: live.aligned,
|
|
pipelineRun: live.pipelineRun,
|
|
inFlightJob: live.inFlightJob,
|
|
message: live.message,
|
|
};
|
|
}
|
|
|
|
function compactFollowerDecision(state: FollowerState): Record<string, unknown> {
|
|
return {
|
|
phase: state.phase,
|
|
observedSha: state.source.observedSha,
|
|
targetSha: state.target.targetSha,
|
|
lastTriggeredSha: state.lastTriggeredSha,
|
|
lastSucceededSha: state.lastSucceededSha,
|
|
pipelineRun: state.pipelineRun,
|
|
inFlightJob: state.inFlightJob,
|
|
decision: state.decision,
|
|
timings: state.timings,
|
|
};
|
|
}
|
|
|
|
function debugNext(followerId: string): Record<string, string> {
|
|
return {
|
|
stateRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${followerId} --step state-read`,
|
|
statusRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${followerId} --step status-read`,
|
|
decide: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${followerId} --step decide`,
|
|
stateWrite: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${followerId} --step state-write --confirm`,
|
|
};
|
|
}
|
|
|
|
function parseLastJsonObject(text: string): Record<string, unknown> | null {
|
|
const starts: number[] = [];
|
|
let offset = 0;
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
if (line.trimStart().startsWith("{")) starts.push(offset + line.indexOf("{"));
|
|
offset += line.length + 1;
|
|
}
|
|
const end = text.lastIndexOf("}");
|
|
if (end < 0) return null;
|
|
for (let index = starts.length - 1; index >= 0; index -= 1) {
|
|
const start = starts[index] ?? -1;
|
|
if (start < 0 || start >= end) continue;
|
|
try {
|
|
const parsed = JSON.parse(text.slice(start, end + 1)) as unknown;
|
|
const record = asOptionalRecord(parsed);
|
|
if (record !== null) return record;
|
|
} catch {
|
|
// Try the previous JSON-looking line.
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function asOptionalRecord(value: unknown): Record<string, unknown> | null {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
|
}
|
|
|
|
function stringOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function numberOrNull(value: unknown): number | null {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
function shortSha(value: string | null): string {
|
|
if (value === null) return "-";
|
|
return value.length > 12 ? value.slice(0, 12) : value;
|
|
}
|
|
|
|
function table(headers: readonly string[], rows: readonly (readonly unknown[])[]): string {
|
|
const normalized = rows.map((row) => headers.map((_, index) => cell(row[index])));
|
|
const widths = headers.map((header, index) => Math.max(header.length, ...normalized.map((row) => row[index]?.length ?? 0)));
|
|
const format = (row: readonly string[]) => row.map((value, index) => value.padEnd(widths[index] ?? 0)).join(" ").trimEnd();
|
|
return [format(headers), format(headers.map((header) => "-".repeat(header.length))), ...normalized.map(format)].join("\n");
|
|
}
|
|
|
|
function cell(value: unknown): string {
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
const text = String(value).replace(/\s+/gu, " ");
|
|
return text.length > 96 ? `${text.slice(0, 93)}...` : text;
|
|
}
|
|
|
|
function tailText(text: string, maxChars: number): string {
|
|
return text.length <= maxChars ? text : text.slice(text.length - maxChars);
|
|
}
|