diff --git a/scripts/src/cicd.ts b/scripts/src/cicd.ts index 9221f5a5..4455adae 100644 --- a/scripts/src/cicd.ts +++ b/scripts/src/cicd.ts @@ -31,7 +31,7 @@ const SPEC_REF = "PJ2026-01060703"; const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower"; type OutputMode = "human" | "json" | "yaml"; -type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "events" | "logs"; +type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "cleanup-state" | "events" | "logs"; type BranchFollowerPhase = | "Observed" | "Noop" @@ -209,6 +209,7 @@ interface AdapterSummary { lastTriggeredSha: string | null; lastSucceededSha: string | null; pipelineRun: string | null; + pipelineRunPresent: boolean | null; inFlightJob: string | null; aligned: boolean | null; phase: BranchFollowerPhase; @@ -286,9 +287,16 @@ interface K8sStateRead { errors: string[]; } +interface K8sFollowerStateRead { + ok: boolean; + stateByFollower: Record>; + present: boolean; + error: string; +} + export function cicdHelp(): unknown { return { - command: "cicd branch-follower plan|apply|status|run-once|events|logs", + command: "cicd branch-follower plan|apply|status|run-once|cleanup-state|events|logs", output: "text by default; use --json, --raw, or -o json|yaml for machine output", usage: [ "bun scripts/cli.ts cicd branch-follower plan", @@ -297,6 +305,7 @@ export function cicdHelp(): unknown { "bun scripts/cli.ts cicd branch-follower status --live", "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", "bun scripts/cli.ts cicd branch-follower run-once --follower hwlab-jd01-v03 --confirm --wait", + "bun scripts/cli.ts cicd branch-follower cleanup-state --follower web-probe-sentinel-master --confirm", "bun scripts/cli.ts cicd branch-follower events --follower agentrun-jd01-v02", "bun scripts/cli.ts cicd branch-follower logs --follower web-probe-sentinel-master", ], @@ -310,7 +319,7 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string const top = args[0]; if (top === undefined || isHelpToken(top)) return renderMachine("cicd", cicdHelp(), "json"); if (top !== "branch-follower") { - throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|events|logs"); + throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|cleanup-state|events|logs"); } const options = parseOptions(args.slice(1)); const command = commandLabel(options); @@ -325,6 +334,8 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string return renderResult(command, await buildStatus(registry, options), options); case "run-once": return renderResult(command, await runOnce(registry, options), options); + case "cleanup-state": + return renderResult(command, cleanupState(registry, options), options); case "events": case "logs": return renderResult(command, await runFollowerDrillDown(registry, options), options); @@ -338,7 +349,7 @@ function parseOptions(args: string[]): ParsedOptions { if (actionToken === undefined || isHelpToken(actionToken)) { return defaultOptions("help", args.slice(actionToken === undefined ? 0 : 1)); } - if (!["plan", "apply", "status", "run-once", "events", "logs"].includes(actionToken)) { + if (!["plan", "apply", "status", "run-once", "cleanup-state", "events", "logs"].includes(actionToken)) { throw new Error(`cicd branch-follower unknown action: ${actionToken}`); } const action = actionToken as BranchFollowerAction; @@ -395,9 +406,13 @@ function parseOptions(args: string[]): ParsedOptions { if (options.confirm && options.dryRun) throw new Error("cicd branch-follower accepts only one of --confirm or --dry-run"); if (options.action === "apply" && !options.confirm) options.dryRun = true; if (options.action === "run-once" && !options.confirm) options.dryRun = true; + if (options.action === "cleanup-state" && !options.confirm) options.dryRun = true; if (options.action === "run-once" && options.confirm && !options.all && options.followerId === null) { throw new Error("run-once --confirm requires --all or --follower "); } + if (options.action === "cleanup-state" && options.confirm && !options.all && options.followerId === null) { + throw new Error("cleanup-state --confirm requires --all or --follower "); + } return options; } @@ -849,6 +864,43 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions) }; } +function cleanupState(registry: BranchFollowerRegistry, options: ParsedOptions): Record { + const selected = selectFollowers(registry, options, { includeDisabled: true }); + const ids = selected.map((follower) => follower.id); + const before = kubeConfigMapFollowerState(registry, options); + const presentKeys = ids.filter((id) => before.stateByFollower[id] !== undefined); + const command = options.confirm ? removeFollowerStateKeys(registry, options, ids) : null; + return { + ok: command === null ? before.ok : command.exitCode === 0, + action: "cleanup-state", + dryRun: !options.confirm, + confirm: options.confirm, + execution: "k8s-state-configmap-key-delete", + registry: registrySummary(registry), + controller: { + namespace: registry.controller.namespace, + route: registry.controller.kubeRoute, + stateConfigMapName: registry.controller.stateConfigMapName, + }, + followers: ids.map((id) => ({ + id, + statePresent: before.stateByFollower[id] !== undefined, + cleanup: options.confirm ? "deleted-if-present" : "would-delete-if-present", + })), + stateConfigMapPresent: before.present, + presentKeys, + parsedDownstreamCliOutput: false, + command: command === null ? null : commandCompact(command, options), + errors: before.ok ? [] : [before.error], + next: { + status: "bun scripts/cli.ts cicd branch-follower status", + runOnce: options.followerId === null + ? "bun scripts/cli.ts cicd branch-follower run-once --all --confirm --wait" + : `bun scripts/cli.ts cicd branch-follower run-once --follower ${options.followerId} --confirm --wait`, + }, + }; +} + async function runFollowerDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { if (options.followerId === null) { return { @@ -939,7 +991,10 @@ async function decideAndMaybeTrigger( const previousObserved = stringOrNull(recordAt(previous, ["source"])?.observedSha); const previousTarget = stringOrNull(recordAt(previous, ["target"])?.targetSha); const superseded = previousInFlight !== null && previousObserved !== null && observedSha !== null && previousObserved !== observedSha; - const nativePipelineRunMatchesObserved = observedSha !== null && live.pipelineRun !== null && live.pipelineRun === expectedPipelineRunName(follower, observedSha); + const nativePipelineRunMatchesObserved = observedSha !== null + && live.pipelineRunPresent === true + && live.pipelineRun !== null + && live.pipelineRun === expectedPipelineRunName(follower, observedSha); let phase: BranchFollowerPhase; let decision: string; let triggerCommand: Record | undefined; @@ -1271,6 +1326,7 @@ async function readAdapterStatus(registry: BranchFollowerRegistry, follower: Fol const observedSha = stringOrNull(bundle.source?.commit); const runtimeTargetSha = runtimeTargetShaFromWorkloads(follower.nativeStatus.runtime, bundle.workloads); const pipelineRunName = stringOrNull(asOptionalRecord(bundle.pipelineRun?.metadata)?.name) ?? expectedPipelineRunName(follower, observedSha); + const pipelineRunPresent = follower.nativeStatus.tekton === null ? null : bundle.pipelineRun !== null; const pipelineSucceeded = pipelineRunSucceeded(bundle.pipelineRun); const argoReady = follower.nativeStatus.argo === null ? null : argoApplicationReady(bundle.argoApplication); const runtimeReady = follower.nativeStatus.runtime === null ? null : runtimeWorkloadsReady(follower.nativeStatus.runtime, bundle.workloads); @@ -1300,7 +1356,8 @@ async function readAdapterStatus(registry: BranchFollowerRegistry, follower: Fol lastTriggeredSha: null, lastSucceededSha: aligned && observedSha !== null ? observedSha : null, pipelineRun: pipelineRunName, - inFlightJob: pipelineSucceeded === null && pipelineRunName !== null ? pipelineRunName : null, + pipelineRunPresent, + inFlightJob: pipelineRunPresent === true && pipelineSucceeded === null && pipelineRunName !== null ? pipelineRunName : null, aligned, phase, message: nativeStatusMessage(ok, phase, observedSha, targetSha, { @@ -1748,29 +1805,19 @@ function mergeFollowerStatus( function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead { const errors: string[] = []; - const stateResult = kubeJson(registry, options, `kubectl -n ${shQuote(registry.controller.namespace)} get configmap ${shQuote(registry.controller.stateConfigMapName)} -o json`, 10_000); + const stateResult = kubeConfigMapFollowerState(registry, options); const deploymentResult = kubeJson(registry, options, `kubectl -n ${shQuote(registry.controller.namespace)} get deploy ${shQuote(registry.controller.deploymentName)} -o json`, 10_000); const leaseResult = kubeJson(registry, options, `kubectl -n ${shQuote(registry.controller.namespace)} get lease ${shQuote(registry.controller.leaseName)} -o json`, 10_000); const podSelector = labelSelector(registry.controller.labels); const podsResult = kubePodList(registry, options, podSelector); - if (!stateResult.ok && !isNotFoundText(stateResult.error)) errors.push(`state configmap: ${stateResult.error}`); + if (!stateResult.ok) errors.push(`state configmap: ${stateResult.error}`); if (!deploymentResult.ok && !isNotFoundText(deploymentResult.error)) errors.push(`deployment: ${deploymentResult.error}`); if (!leaseResult.ok && !isNotFoundText(leaseResult.error)) errors.push(`lease: ${leaseResult.error}`); if (!podsResult.ok && !isNotFoundText(podsResult.error)) errors.push(`pods: ${podsResult.error}`); - const stateByFollower: Record> = {}; - const data = asOptionalRecord(stateResult.value?.data); - if (data !== null) { - for (const [key, value] of Object.entries(data)) { - if (key.startsWith("_")) continue; - if (typeof value !== "string") continue; - const parsed = parseJsonObject(value); - if (parsed !== null) stateByFollower[key] = parsed; - } - } return { ok: errors.length === 0, - stateByFollower, - stateConfigMapPresent: stateResult.value !== null, + stateByFollower: stateResult.stateByFollower, + stateConfigMapPresent: stateResult.present, deployment: deploymentResult.value, lease: leaseResult.value, pods: podsResult.value, @@ -1778,6 +1825,106 @@ function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): }; } +function kubeConfigMapFollowerState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sFollowerStateRead { + const stateByFollower: Record> = {}; + const errors: string[] = []; + let present = false; + for (const follower of registry.followers) { + const result = kubeConfigMapDataValue(registry, options, follower.id); + if (!result.ok) { + errors.push(result.error); + continue; + } + if (!result.present) { + return { ok: true, stateByFollower: {}, present: false, error: "" }; + } + present = true; + if (result.value === null || result.value.length === 0) continue; + const parsed = parseJsonObject(result.value); + if (parsed === null) { + errors.push(`${follower.id}: invalid state json`); + continue; + } + stateByFollower[follower.id] = parsed; + } + return { + ok: errors.length === 0, + stateByFollower, + present, + error: errors.join("; "), + }; +} + +function kubeConfigMapDataValue(registry: BranchFollowerRegistry, options: ParsedOptions, key: string): { ok: boolean; present: boolean; value: string | null; error: string } { + const template = `{{ with index .data ${JSON.stringify(key)} }}{{ . }}{{ end }}`; + const script = [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + `NAMESPACE=${shQuote(registry.controller.namespace)}`, + `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, + "export NAMESPACE CONFIGMAP", + `if ! value=$(kubectl -n "$NAMESPACE" get configmap "$CONFIGMAP" -o go-template=${shQuote(template)} 2>"$tmpdir/error"); then`, + " error_b64=$(tail -c 800 \"$tmpdir/error\" | base64 | tr -d '\\n')", + " if grep -qi 'not found' \"$tmpdir/error\"; then", + " printf '{\"ok\":true,\"present\":false,\"valueB64\":null,\"errorB64\":\"%s\"}' \"$error_b64\"", + " exit 0", + " fi", + " printf '{\"ok\":false,\"present\":false,\"valueB64\":null,\"errorB64\":\"%s\"}' \"$error_b64\"", + " exit 0", + "fi", + "value_b64=$(printf '%s' \"$value\" | base64 | tr -d '\\n')", + "printf '{\"ok\":true,\"present\":true,\"valueB64\":\"%s\",\"errorB64\":\"\"}' \"$value_b64\"", + ].join("\n"); + const result = runKubeScript(registry, options, script, "", 10_000); + const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; + if (parsed === null) { + return { + ok: false, + present: false, + value: null, + error: redactText(tailText(result.stderr || result.stdout, 800)), + }; + } + const errorB64 = typeof parsed.errorB64 === "string" ? parsed.errorB64 : ""; + const error = errorB64.length === 0 ? "" : Buffer.from(errorB64, "base64").toString("utf8"); + const ok = parsed.ok === true; + const present = parsed.present === true; + const valueB64 = typeof parsed.valueB64 === "string" ? parsed.valueB64 : null; + return { + ok, + present, + value: valueB64 === null ? null : Buffer.from(valueB64, "base64").toString("utf8"), + error: redactText(error), + }; +} + +function removeFollowerStateKeys(registry: BranchFollowerRegistry, options: ParsedOptions, ids: string[]): CommandResult { + const patch = JSON.stringify({ data: Object.fromEntries(ids.map((id) => [id, null])) }); + const script = [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + `NAMESPACE=${shQuote(registry.controller.namespace)}`, + `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, + `PATCH=${shQuote(patch)}`, + "export NAMESPACE CONFIGMAP PATCH", + "if ! kubectl -n \"$NAMESPACE\" get configmap \"$CONFIGMAP\" >/dev/null 2>\"$tmpdir/error\"; then", + " if grep -qi 'not found' \"$tmpdir/error\"; then", + " printf '{\"ok\":true,\"present\":false,\"patched\":false,\"reason\":\"state-configmap-not-found\",\"parsedDownstreamCliOutput\":false}'", + " exit 0", + " fi", + " cat \"$tmpdir/error\" >&2", + " exit 1", + "fi", + "kubectl -n \"$NAMESPACE\" patch configmap \"$CONFIGMAP\" --type merge -p \"$PATCH\" >/dev/null", + "printf '{\"ok\":true,\"present\":true,\"patched\":true,\"parsedDownstreamCliOutput\":false}'", + ].join("\n"); + return runKubeScript(registry, options, script, "", 10_000); +} + function kubeJson(registry: BranchFollowerRegistry, options: ParsedOptions, command: string, timeoutMs: number): { ok: boolean; value: Record | null; error: string } { const result = runKubeScript(registry, options, `set -eu\n${command}`, "", timeoutMs); const value = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; @@ -2440,6 +2587,7 @@ function renderHuman(command: string, payload: Record, options: if (command.endsWith(" apply")) return renderApplyHuman(payload); if (command.endsWith(" status")) return renderStatusHuman(payload, options); if (command.endsWith(" run-once")) return renderRunOnceHuman(payload); + if (command.endsWith(" cleanup-state")) return renderCleanupStateHuman(payload); if (command.endsWith(" events") || command.endsWith(" logs")) return renderDrillDownHuman(payload); return `${JSON.stringify(payload, null, 2)}\n`; } @@ -2568,6 +2716,34 @@ function renderRunOnceHuman(payload: Record): string { ].join("\n"); } +function renderCleanupStateHuman(payload: Record): string { + const controller = asOptionalRecord(payload.controller); + const command = asOptionalRecord(payload.command); + const followers = arrayRecords(payload.followers); + const next = asOptionalRecord(payload.next); + const rows = followers.map((item) => [ + item.id, + item.statePresent === true ? "present" : "missing", + item.cleanup ?? "-", + ]); + return [ + `CI/CD BRANCH-FOLLOWER CLEANUP-STATE (${payload.ok === false ? "failed" : payload.dryRun === true ? "dry-run" : "ok"})`, + "", + table( + ["NAMESPACE", "ROUTE", "STATE_CM", "STATE_CM_PRESENT"], + [[controller?.namespace ?? "-", controller?.route ?? "-", controller?.stateConfigMapName ?? "-", payload.stateConfigMapPresent === true ? "present" : "missing"]], + ), + "", + table(["FOLLOWER", "STATE", "CLEANUP"], rows), + command === null ? "" : `\nPATCH\nexit=${command.exitCode ?? "-"} timedOut=${command.timedOut ?? "-"}`, + "", + "NEXT", + `status: ${next?.status ?? "-"}`, + `run-once: ${next?.runOnce ?? "-"}`, + "", + ].filter((line) => line !== "").join("\n"); +} + function renderDrillDownHuman(payload: Record): string { if (payload.follower === undefined) { const followers = arrayRecords(payload.followers);