feat: add branch follower debug step

2026-07-03 17:56:08 +00:00
parent b98b131bb7
commit 917cba6659
8 changed files with 486 additions and 8 deletions
@@ -4,11 +4,11 @@ import { createHash } from "node:crypto";
 import { readFileSync } from "node:fs";
 import { rootPath } from "./config";
 import { shQuote } from "./platform-infra-ops-library";
-import type { BranchFollowerRegistry, ParsedOptions } from "./cicd-types";
+import type { BranchFollowerDebugStep, BranchFollowerRegistry, ParsedOptions } from "./cicd-types";

 const SPEC_REF = "PJ2026-01060703";

-export function renderControllerReconcileJob(registry: BranchFollowerRegistry, options: ParsedOptions, jobName: string, mode: { dryRun: boolean; recordState: boolean }, timeoutSeconds: number): Record<string, unknown> {
+export function renderControllerReconcileJob(registry: BranchFollowerRegistry, options: ParsedOptions, jobName: string, mode: { dryRun: boolean; wait?: boolean; recordState: boolean }, timeoutSeconds: number): Record<string, unknown> {
  const labels = { ...registry.controller.labels, "app.kubernetes.io/component": "cicd-reconcile-job" };
  const commandArgs = [
    "bun",
@@ -74,6 +74,75 @@ export function renderControllerReconcileJob(registry: BranchFollowerRegistry, o
  };
 }

+export function renderControllerDebugJob(registry: BranchFollowerRegistry, options: ParsedOptions, jobName: string, step: BranchFollowerDebugStep, timeoutSeconds: number): Record<string, unknown> {
+  if (options.followerId === null) throw new Error("debug-step target job requires --follower <id>");
+  const labels = { ...registry.controller.labels, "app.kubernetes.io/component": "cicd-debug-job" };
+  const commandArgs = [
+    "bun",
+    "scripts/cli.ts",
+    "cicd",
+    "branch-follower",
+    "debug-step",
+    "--follower",
+    options.followerId,
+    "--step",
+    step,
+    options.confirm ? "--confirm" : "--dry-run",
+    "--in-cluster",
+    "--config",
+    "config/cicd-branch-followers.yaml",
+    "--timeout-seconds",
+    String(timeoutSeconds),
+    "--json",
+  ];
+  return {
+    apiVersion: "batch/v1",
+    kind: "Job",
+    metadata: { name: jobName, namespace: registry.controller.namespace, labels },
+    spec: {
+      backoffLimit: registry.controller.budgets.reconcileJobBackoffLimit,
+      ttlSecondsAfterFinished: registry.controller.budgets.reconcileJobTtlSeconds,
+      activeDeadlineSeconds: timeoutSeconds + registry.controller.budgets.reconcileJobDeadlineGraceSeconds,
+      template: {
+        metadata: { labels },
+        spec: {
+          restartPolicy: "Never",
+          serviceAccountName: registry.controller.serviceAccountName,
+          volumes: [
+            { name: "registry", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } },
+            { name: "git-mirror-cache", persistentVolumeClaim: { claimName: registry.controller.source.gitMirrorCachePvcName } },
+            { name: "git-ssh", secret: { secretName: registry.controller.source.githubSsh.secretName, defaultMode: 0o400 } },
+            { name: "work", emptyDir: {} },
+          ],
+          containers: [
+            {
+              name: "debug",
+              image: registry.controller.image,
+              imagePullPolicy: "IfNotPresent",
+              command: ["/bin/sh", "/etc/unidesk-cicd-branch-follower/controller-one-shot.sh"],
+              args: commandArgs,
+              env: [
+                { name: "UNIDESK_CONTROLLER_SOURCE_BRANCH", value: registry.controller.source.branch },
+                { name: "UNIDESK_CONTROLLER_SOURCE_REPOSITORY", value: registry.controller.source.repository },
+                { name: "UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX", value: registry.controller.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", registry.controller.source.branch) },
+                { name: "UNIDESK_CONTROLLER_GITHUB_SSH_PRIVATE_KEY", value: `/git-ssh/${registry.controller.source.githubSsh.privateKeySecretKey}` },
+                { name: "UNIDESK_CONTROLLER_GITHUB_PROXY_HOST", value: registry.controller.source.githubSsh.proxyHost },
+                { name: "UNIDESK_CONTROLLER_GITHUB_PROXY_PORT", value: String(registry.controller.source.githubSsh.proxyPort) },
+              ],
+              volumeMounts: [
+                { name: "registry", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true },
+                { name: "git-mirror-cache", mountPath: "/cache" },
+                { name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
+                { name: "work", mountPath: "/work" },
+              ],
+            },
+          ],
+        },
+      },
+    },
+  };
+}
+
 export function waitForJobShell(namespace: string, jobName: string, timeoutSeconds: number): string {
  return [
    `NAMESPACE=${shQuote(namespace)}`,
@@ -0,0 +1,309 @@
+// SPEC: PJ2026-01060703 CI/CD branch follower debug steps.
+// Responsibility: bounded single-step debugging for branch follower state and decision paths.
+import type { CommandResult } from "./command";
+import type { AdapterSummary, BranchFollowerDebugStep, BranchFollowerRegistry, FollowerSpec, FollowerState, K8sStateRead, ParsedOptions } from "./cicd-types";
+import { renderControllerDebugJob, waitForJobShell } from "./cicd-controller-render";
+import { redactText, shQuote } from "./platform-infra-ops-library";
+
+type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult;
+
+export interface CicdDebugDeps {
+  selectFollowers(registry: BranchFollowerRegistry, options: ParsedOptions, opts: { includeDisabled: boolean }): FollowerSpec[];
+  readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead;
+  readAdapterStatus(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Promise<AdapterSummary>;
+  decideAndMaybeTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, previous: Record<string, unknown>, live: AdapterSummary, options: ParsedOptions): Promise<FollowerState>;
+  writeFollowerState(registry: BranchFollowerRegistry, state: FollowerState, options: ParsedOptions): CommandResult;
+  runKubeScript: KubeScriptRunner;
+}
+
+export async function buildDebugStep(registry: BranchFollowerRegistry, options: ParsedOptions, deps: CicdDebugDeps): Promise<Record<string, unknown>> {
+  const step = options.debugStep ?? "state-read";
+  if (options.followerId === null) throw new Error("debug-step requires --follower <id>");
+  if (!options.inCluster) return runTargetDebugStepJob(registry, options, step, deps);
+
+  const selected = deps.selectFollowers(registry, options, { includeDisabled: true });
+  if (selected.length !== 1) throw new Error("debug-step operates on exactly one follower");
+  const follower = selected[0] as FollowerSpec;
+  const before = deps.readK8sState(registry, options);
+  const previous = before.stateByFollower[follower.id] ?? {};
+  let live: AdapterSummary | null = null;
+  let decided: FollowerState | null = null;
+  let write: Record<string, unknown> | null = null;
+  let after: K8sStateRead | null = null;
+
+  if (step === "status-read" || step === "decide") {
+    live = await deps.readAdapterStatus(registry, follower, options);
+  }
+  if (step === "decide") {
+    decided = await deps.decideAndMaybeTrigger(registry, follower, previous, live as AdapterSummary, debugDecisionOptions(options));
+  }
+  if (step === "state-write") {
+    const writeInput = stateWriteInput(previous);
+    if (writeInput === null) {
+      return {
+        ok: false,
+        action: "debug-step",
+        step,
+        follower: follower.id,
+        execution: "k8s-native-in-cluster",
+        dryRun: !options.confirm,
+        stateBefore: stateSnapshot(before, follower.id),
+        stateWrite: { ok: false, skipped: true, reason: "stored-state-missing" },
+        parsedDownstreamCliOutput: false,
+      };
+    }
+    if (options.confirm) {
+      const result = deps.writeFollowerState(registry, writeInput, options);
+      write = stateWriteResult(follower.id, result);
+      after = deps.readK8sState(registry, options);
+    } else {
+      write = { ok: true, skipped: true, reason: "dry-run-requires-confirm", input: "stored-state" };
+    }
+  }
+
+  return {
+    ok: write === null ? before.ok && (live === null || live.ok) : write.ok === true,
+    action: "debug-step",
+    step,
+    follower: follower.id,
+    execution: "k8s-native-in-cluster",
+    dryRun: !options.confirm,
+    stateBefore: stateSnapshot(before, follower.id),
+    status: live === null ? null : compactAdapterStatus(live),
+    decision: decided === null ? null : compactFollowerDecision(decided),
+    stateWrite: write,
+    stateAfter: after === null ? null : stateSnapshot(after, follower.id),
+    parsedDownstreamCliOutput: false,
+    next: debugNext(follower.id),
+  };
+}
+
+export function renderDebugStepHuman(payload: Record<string, unknown>): string {
+  const before = asOptionalRecord(payload.stateBefore);
+  const after = asOptionalRecord(payload.stateAfter);
+  const write = asOptionalRecord(payload.stateWrite);
+  const status = asOptionalRecord(payload.status);
+  const decision = asOptionalRecord(payload.decision);
+  const target = asOptionalRecord(payload.target);
+  const next = asOptionalRecord(payload.next);
+  const rows = [[
+    payload.follower ?? "-",
+    payload.step ?? "-",
+    payload.execution ?? "-",
+    payload.dryRun === true ? "true" : "false",
+    before?.phase ?? "-",
+    after?.phase ?? decision?.phase ?? status?.phase ?? "-",
+    shortSha(stringOrNull(before?.observedSha)),
+    shortSha(stringOrNull(after?.observedSha) ?? stringOrNull(decision?.observedSha) ?? stringOrNull(status?.observedSha)),
+  ]];
+  const writeRows = write === null ? [] : [[
+    write.ok === true ? "ok" : "failed",
+    write.skipped === true ? "skipped" : "executed",
+    write.input ?? "-",
+    asOptionalRecord(write.patch)?.beforeResourceVersion ?? "-",
+    asOptionalRecord(write.patch)?.afterResourceVersion ?? "-",
+    write.exitCode ?? "-",
+    write.message ?? write.reason ?? "-",
+  ]];
+  return [
+    `CI/CD BRANCH-FOLLOWER DEBUG-STEP (${payload.ok === false ? "failed" : "ok"})`,
+    "",
+    table(["FOLLOWER", "STEP", "EXECUTION", "DRY_RUN", "BEFORE", "AFTER", "BEFORE_SHA", "AFTER_SHA"], rows),
+    target === null ? "" : `\nTARGET JOB\n${table(["JOB", "EXIT", "TIMED_OUT", "PARSED"], [[target.name ?? "-", target.exitCode ?? "-", target.timedOut ?? "-", target.parsed === true ? "yes" : "no"]])}`,
+    writeRows.length === 0 ? "" : `\nSTATE WRITE\n${table(["STATUS", "MODE", "INPUT", "BEFORE_RV", "AFTER_RV", "EXIT", "MESSAGE"], writeRows)}`,
+    "",
+    "NEXT",
+    `state-read: ${next?.stateRead ?? "-"}`,
+    `status-read: ${next?.statusRead ?? "-"}`,
+    `decide: ${next?.decide ?? "-"}`,
+    `state-write: ${next?.stateWrite ?? "-"}`,
+    "",
+  ].filter((line) => line !== "").join("\n");
+}
+
+function runTargetDebugStepJob(registry: BranchFollowerRegistry, options: ParsedOptions, step: BranchFollowerDebugStep, deps: CicdDebugDeps): Record<string, unknown> {
+  const timeoutSeconds = options.timeoutSeconds ?? registry.controller.budgets.runOnceSeconds;
+  const jobName = `${registry.controller.deploymentName}-debug-${step}-${Date.now().toString(36)}`.replace(/[^a-z0-9-]+/gu, "-").slice(0, 63);
+  const manifest = renderControllerDebugJob(registry, options, jobName, step, timeoutSeconds);
+  const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
+  const script = [
+    "set -eu",
+    "tmp=$(mktemp)",
+    "base64 -d >\"$tmp\" <<'UNIDESK_CICD_DEBUG_JOB_B64'",
+    Buffer.from(manifestYaml, "utf8").toString("base64"),
+    "UNIDESK_CICD_DEBUG_JOB_B64",
+    `kubectl -n ${shQuote(registry.controller.namespace)} delete job ${shQuote(jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`,
+    `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`,
+    waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds),
+  ].join("\n");
+  const result = deps.runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000);
+  const parsed = parseLastJsonObject(result.stdout);
+  const state = deps.readK8sState(registry, options);
+  const followerId = options.followerId ?? "";
+  return {
+    ok: result.exitCode === 0 && parsed?.ok !== false,
+    action: "debug-step",
+    step,
+    follower: followerId,
+    execution: "k8s-native-debug-job",
+    dryRun: !options.confirm,
+    stateBefore: asOptionalRecord(parsed?.stateBefore),
+    status: asOptionalRecord(parsed?.status),
+    decision: asOptionalRecord(parsed?.decision),
+    stateWrite: asOptionalRecord(parsed?.stateWrite),
+    target: {
+      name: jobName,
+      namespace: registry.controller.namespace,
+      exitCode: result.exitCode,
+      timedOut: result.timedOut,
+      parsed: parsed !== null,
+      stdoutTail: redactText(tailText(result.stdout, options.full ? 4000 : 1000)),
+      stderrTail: redactText(tailText(result.stderr, options.full ? 2000 : 800)),
+    },
+    targetResult: parsed,
+    stateAfter: asOptionalRecord(parsed?.stateAfter) ?? stateSnapshot(state, followerId),
+    parsedDownstreamCliOutput: false,
+    next: debugNext(followerId),
+  };
+}
+
+function debugDecisionOptions(options: ParsedOptions): ParsedOptions {
+  return { ...options, confirm: false, dryRun: true, wait: false, recordState: false };
+}
+
+function stateWriteInput(previous: Record<string, unknown>): FollowerState | null {
+  if (stringOrNull(previous.id) === null) return null;
+  if (asOptionalRecord(previous.source) === null || asOptionalRecord(previous.target) === null || asOptionalRecord(previous.timings) === null) return null;
+  return previous as unknown as FollowerState;
+}
+
+function stateWriteResult(followerId: string, result: CommandResult): Record<string, unknown> {
+  const parsed = parseLastJsonObject(result.stdout);
+  return {
+    ok: result.exitCode === 0 && parsed?.ok !== false,
+    follower: followerId,
+    exitCode: result.exitCode,
+    timedOut: result.timedOut,
+    input: "stored-state",
+    patch: parsed,
+    message: result.exitCode === 0 ? "state patch command completed" : redactText(tailText(result.stderr || result.stdout, 500)),
+    parsedDownstreamCliOutput: false,
+  };
+}
+
+function stateSnapshot(read: K8sStateRead, followerId: string): Record<string, unknown> {
+  const state = read.stateByFollower[followerId] ?? {};
+  const source = asOptionalRecord(state.source);
+  const target = asOptionalRecord(state.target);
+  const timings = asOptionalRecord(state.timings);
+  return {
+    present: read.stateConfigMapPresent,
+    ok: read.ok,
+    metadata: read.stateMetadata,
+    valueBytes: read.stateValueBytes[followerId] ?? null,
+    phase: stringOrNull(state.phase),
+    observedSha: stringOrNull(source?.observedSha),
+    targetSha: stringOrNull(target?.targetSha),
+    lastTriggeredSha: stringOrNull(state.lastTriggeredSha),
+    lastSucceededSha: stringOrNull(state.lastSucceededSha),
+    pipelineRun: stringOrNull(state.pipelineRun),
+    inFlightJob: stringOrNull(state.inFlightJob),
+    timingStatus: stringOrNull(timings?.totalStatus),
+    totalSeconds: numberOrNull(timings?.totalSeconds),
+    startedAt: stringOrNull(timings?.startedAt),
+    updatedAt: stringOrNull(state.updatedAt),
+  };
+}
+
+function compactAdapterStatus(live: AdapterSummary): Record<string, unknown> {
+  return {
+    ok: live.ok,
+    phase: live.phase,
+    observedSha: live.observedSha,
+    targetSha: live.targetSha,
+    aligned: live.aligned,
+    pipelineRun: live.pipelineRun,
+    inFlightJob: live.inFlightJob,
+    message: live.message,
+  };
+}
+
+function compactFollowerDecision(state: FollowerState): Record<string, unknown> {
+  return {
+    phase: state.phase,
+    observedSha: state.source.observedSha,
+    targetSha: state.target.targetSha,
+    lastTriggeredSha: state.lastTriggeredSha,
+    lastSucceededSha: state.lastSucceededSha,
+    pipelineRun: state.pipelineRun,
+    inFlightJob: state.inFlightJob,
+    decision: state.decision,
+    timings: state.timings,
+  };
+}
+
+function debugNext(followerId: string): Record<string, string> {
+  return {
+    stateRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${followerId} --step state-read`,
+    statusRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${followerId} --step status-read`,
+    decide: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${followerId} --step decide`,
+    stateWrite: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${followerId} --step state-write --confirm`,
+  };
+}
+
+function parseLastJsonObject(text: string): Record<string, unknown> | null {
+  const starts: number[] = [];
+  let offset = 0;
+  for (const line of text.split(/\r?\n/u)) {
+    if (line.trimStart().startsWith("{")) starts.push(offset + line.indexOf("{"));
+    offset += line.length + 1;
+  }
+  const end = text.lastIndexOf("}");
+  if (end < 0) return null;
+  for (let index = starts.length - 1; index >= 0; index -= 1) {
+    const start = starts[index] ?? -1;
+    if (start < 0 || start >= end) continue;
+    try {
+      const parsed = JSON.parse(text.slice(start, end + 1)) as unknown;
+      const record = asOptionalRecord(parsed);
+      if (record !== null) return record;
+    } catch {
+      // Try the previous JSON-looking line.
+    }
+  }
+  return null;
+}
+
+function asOptionalRecord(value: unknown): Record<string, unknown> | null {
+  return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
+}
+
+function stringOrNull(value: unknown): string | null {
+  return typeof value === "string" && value.length > 0 ? value : null;
+}
+
+function numberOrNull(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function shortSha(value: string | null): string {
+  if (value === null) return "-";
+  return value.length > 12 ? value.slice(0, 12) : value;
+}
+
+function table(headers: readonly string[], rows: readonly (readonly unknown[])[]): string {
+  const normalized = rows.map((row) => headers.map((_, index) => cell(row[index])));
+  const widths = headers.map((header, index) => Math.max(header.length, ...normalized.map((row) => row[index]?.length ?? 0)));
+  const format = (row: readonly string[]) => row.map((value, index) => value.padEnd(widths[index] ?? 0)).join("  ").trimEnd();
+  return [format(headers), format(headers.map((header) => "-".repeat(header.length))), ...normalized.map(format)].join("\n");
+}
+
+function cell(value: unknown): string {
+  if (value === null || value === undefined || value === "") return "-";
+  const text = String(value).replace(/\s+/gu, " ");
+  return text.length > 96 ? `${text.slice(0, 93)}...` : text;
+}
+
+function tailText(text: string, maxChars: number): string {
+  return text.length <= maxChars ? text : text.slice(text.length - maxChars);
+}
@@ -2,7 +2,8 @@
 // Responsibility: type contracts shared by branch follower entry, controller render, and native K8s helpers.

 export type OutputMode = "human" | "json" | "yaml";
-export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "cleanup-state" | "events" | "logs";
+export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "debug-step" | "cleanup-state" | "events" | "logs";
+export type BranchFollowerDebugStep = "state-read" | "status-read" | "decide" | "state-write";
 export type BranchFollowerPhase =
  | "Observed"
  | "Noop"
@@ -29,6 +30,7 @@ export interface ParsedOptions {
  full: boolean;
  raw: boolean;
  recordState: boolean;
+  debugStep: BranchFollowerDebugStep | null;
  output: OutputMode;
  limit: number;
  tailBytes: number;
@@ -317,6 +319,8 @@ export interface FollowerState {
 export interface K8sStateRead {
  ok: boolean;
  stateByFollower: Record<string, Record<string, unknown>>;
+  stateMetadata: Record<string, unknown> | null;
+  stateValueBytes: Record<string, number>;
  stateConfigMapPresent: boolean;
  deployment: Record<string, unknown> | null;
  lease: Record<string, unknown> | null;
@@ -327,6 +331,8 @@ export interface K8sStateRead {
 export interface K8sFollowerStateRead {
  ok: boolean;
  stateByFollower: Record<string, Record<string, unknown>>;
+  metadata: Record<string, unknown> | null;
+  valueBytes: Record<string, number>;
  present: boolean;
  error: string;
 }
@@ -21,10 +21,11 @@ import { sentinelPipelineRunName } from "./hwlab-node-web-sentinel-cicd-shared";
 import { transPath } from "./hwlab-node/runtime-common";
 import { configRefGraph, resolveConfigRefString } from "./ops/config-refs";
 import { renderControllerManifests, renderControllerReconcileJob, waitForJobShell } from "./cicd-controller-render";
+import { buildDebugStep, renderDebugStepHuman } from "./cicd-debug";
 import { runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh";
 import { nativeCicdScriptLoadShell, readNativeObjectBundle } from "./cicd-native-bundle";
 import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native";
-import type { AdapterSummary, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types";
+import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types";
 import {
  arrayField,
  asRecord,
@@ -44,7 +45,7 @@ const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower";

 export function cicdHelp(): unknown {
  return {
-    command: "cicd branch-follower plan|apply|status|run-once|cleanup-state|events|logs",
+    command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs",
    output: "text by default; use --json, --raw, or -o json|yaml for machine output",
    usage: [
      "bun scripts/cli.ts cicd branch-follower plan",
@@ -53,6 +54,8 @@ export function cicdHelp(): unknown {
      "bun scripts/cli.ts cicd branch-follower status --live",
      "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run",
      "bun scripts/cli.ts cicd branch-follower run-once --follower hwlab-jd01-v03 --confirm --wait",
+      "bun scripts/cli.ts cicd branch-follower debug-step --follower web-probe-sentinel-master --step state-read",
+      "bun scripts/cli.ts cicd branch-follower debug-step --follower web-probe-sentinel-master --step state-write --confirm",
      "bun scripts/cli.ts cicd branch-follower cleanup-state --follower web-probe-sentinel-master --confirm",
      "bun scripts/cli.ts cicd branch-follower events --follower agentrun-jd01-v02",
      "bun scripts/cli.ts cicd branch-follower logs --follower web-probe-sentinel-master",
@@ -67,7 +70,7 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string
  const top = args[0];
  if (top === undefined || isHelpToken(top)) return renderMachine("cicd", cicdHelp(), "json");
  if (top !== "branch-follower") {
-    throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|cleanup-state|events|logs");
+    throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs");
  }
  const options = parseOptions(args.slice(1));
  const command = commandLabel(options);
@@ -82,6 +85,8 @@ export async function runCicdCommand(_config: UniDeskConfig | null, args: string
      return renderResult(command, await buildStatus(registry, options), options);
    case "run-once":
      return renderResult(command, await runOnce(registry, options), options);
+    case "debug-step":
+      return renderResult(command, await buildDebugStep(registry, options, { selectFollowers, readK8sState, readAdapterStatus, decideAndMaybeTrigger, writeFollowerState, runKubeScript }), options);
    case "cleanup-state":
      return renderResult(command, cleanupState(registry, options), options);
    case "events":
@@ -97,7 +102,7 @@ function parseOptions(args: string[]): ParsedOptions {
  if (actionToken === undefined || isHelpToken(actionToken)) {
    return defaultOptions("help", args.slice(actionToken === undefined ? 0 : 1));
  }
-  if (!["plan", "apply", "status", "run-once", "cleanup-state", "events", "logs"].includes(actionToken)) {
+  if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs"].includes(actionToken)) {
    throw new Error(`cicd branch-follower unknown action: ${actionToken}`);
  }
  const action = actionToken as BranchFollowerAction;
@@ -136,6 +141,8 @@ function parseOptions(args: string[]): ParsedOptions {
      options.output = "json";
    } else if (arg === "--record-state") {
      options.recordState = true;
+    } else if (arg === "--step") {
+      options.debugStep = debugStepOption(valueOption(rest, ++index, arg));
    } else if (arg === "-o" || arg === "--output") {
      const value = valueOption(rest, ++index, arg);
      if (value !== "json" && value !== "yaml" && value !== "wide" && value !== "text") throw new Error(`${arg} must be json, yaml, wide, or text`);
@@ -157,6 +164,7 @@ function parseOptions(args: string[]): ParsedOptions {
  if (options.confirm && options.dryRun) throw new Error("cicd branch-follower accepts only one of --confirm or --dry-run");
  if (options.action === "apply" && !options.confirm) options.dryRun = true;
  if (options.action === "run-once" && !options.confirm) options.dryRun = true;
+  if (options.action === "debug-step" && !options.confirm) options.dryRun = true;
  if (options.action === "cleanup-state" && !options.confirm) options.dryRun = true;
  if (options.action === "run-once" && options.confirm && !options.all && options.followerId === null) {
    throw new Error("run-once --confirm requires --all or --follower <id>");
@@ -164,9 +172,17 @@ function parseOptions(args: string[]): ParsedOptions {
  if (options.action === "cleanup-state" && options.confirm && !options.all && options.followerId === null) {
    throw new Error("cleanup-state --confirm requires --all or --follower <id>");
  }
+  if (options.action === "debug-step" && options.followerId === null) {
+    throw new Error("debug-step requires --follower <id>");
+  }
  return options;
 }

+function debugStepOption(value: string): BranchFollowerDebugStep {
+  if (value === "state-read" || value === "status-read" || value === "decide" || value === "state-write") return value;
+  throw new Error("--step must be state-read, status-read, decide, or state-write");
+}
+
 function isInClusterRuntime(): boolean {
  return Boolean(process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT);
 }
@@ -186,6 +202,7 @@ function defaultOptions(action: BranchFollowerAction, _args: string[]): ParsedOp
    full: false,
    raw: false,
    recordState: false,
+    debugStep: null,
    output: "human",
    limit: 20,
    tailBytes: 12000,
@@ -599,12 +616,15 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions)
  const previous = readK8sState(registry, options);
  const results: FollowerState[] = [];
  const stateWriteWarnings: string[] = [];
+  const stateWrites: Record<string, unknown>[] = [];
  for (const follower of selected) {
    const oldState = previous.stateByFollower[follower.id] ?? {};
    const live = await readAdapterStatus(registry, follower, options);
    const state = await decideAndMaybeTrigger(registry, follower, oldState, live, options);
    if (!options.dryRun || options.recordState) {
      const write = writeFollowerState(registry, state, options);
+      const writeSummary = stateWriteSummary(follower.id, write);
+      stateWrites.push(writeSummary);
      if (write.exitCode !== 0) {
        const warning = `state write failed for ${follower.id}: ${tailText(write.stderr || write.stdout, 300)}`;
        state.warnings.push(warning);
@@ -622,6 +642,7 @@ async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions)
    controller: options.inCluster,
    registry: registrySummary(registry),
    followers: results,
+    stateWrites,
    warnings: stateWriteWarnings,
    next: {
      status: "bun scripts/cli.ts cicd branch-follower status",
@@ -1847,6 +1868,8 @@ function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions):
  return {
    ok: errors.length === 0,
    stateByFollower: stateResult.stateByFollower,
+    stateMetadata: stateResult.metadata,
+    stateValueBytes: stateResult.valueBytes,
    stateConfigMapPresent: stateResult.present,
    deployment: deploymentResult.value,
    lease: leaseResult.value,
@@ -1876,20 +1899,42 @@ function kubeConfigMapFollowerState(registry: BranchFollowerRegistry, options: P
    return { ok: false, stateByFollower: {}, present: false, error };
  }
  const parsedStates = asOptionalRecord(parsed.stateByFollower) ?? {};
+  const metadata = asOptionalRecord(parsed.metadata);
+  const parsedValueBytes = asOptionalRecord(parsed.valueBytes) ?? {};
  const stateByFollower: Record<string, Record<string, unknown>> = {};
+  const valueBytes: Record<string, number> = {};
  for (const follower of registry.followers) {
    const state = asOptionalRecord(parsedStates[follower.id]);
    if (state !== null) stateByFollower[follower.id] = state;
+    const bytes = numberOrNull(parsedValueBytes[follower.id]);
+    if (bytes !== null) valueBytes[follower.id] = bytes;
  }
  const errors = Array.isArray(parsed.errors) ? parsed.errors.map(String).filter((item) => item.length > 0) : [];
  return {
    ok: parsed.ok === true && errors.length === 0,
    stateByFollower,
+    metadata,
+    valueBytes,
    present: parsed.present === true,
    error: errors.join("; "),
  };
 }

+function stateWriteSummary(followerId: string, result: CommandResult): Record<string, unknown> {
+  const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null;
+  return {
+    follower: followerId,
+    ok: result.exitCode === 0 && parsed?.ok !== false,
+    exitCode: result.exitCode,
+    timedOut: result.timedOut,
+    beforeResourceVersion: stringOrNull(parsed?.beforeResourceVersion),
+    afterResourceVersion: stringOrNull(parsed?.afterResourceVersion),
+    preservedTiming: parsed?.preservedTiming === true,
+    message: result.exitCode === 0 ? "state patch command completed" : redactText(tailText(result.stderr || result.stdout, 500)),
+    parsedDownstreamCliOutput: false,
+  };
+}
+
 function removeFollowerStateKeys(registry: BranchFollowerRegistry, options: ParsedOptions, ids: string[]): CommandResult {
  const patch = JSON.stringify({ data: Object.fromEntries(ids.map((id) => [id, null])) });
  const script = [
@@ -2665,6 +2710,7 @@ function renderHuman(command: string, payload: Record<string, unknown>, options:
  if (command.endsWith(" apply")) return renderApplyHuman(payload);
  if (command.endsWith(" status")) return renderStatusHuman(payload, options);
  if (command.endsWith(" run-once")) return renderRunOnceHuman(payload);
+  if (command.endsWith(" debug-step")) return renderDebugStepHuman(payload);
  if (command.endsWith(" cleanup-state")) return renderCleanupStateHuman(payload);
  if (command.endsWith(" events") || command.endsWith(" logs")) return renderDrillDownHuman(payload);
  return `${JSON.stringify(payload, null, 2)}\n`;
@@ -2770,6 +2816,7 @@ function renderStatusHuman(payload: Record<string, unknown>, _options: ParsedOpt

 function renderRunOnceHuman(payload: Record<string, unknown>): string {
  const followers = arrayRecords(payload.followers);
+  const stateWrites = arrayRecords(payload.stateWrites);
  const rows = followers.map((item) => {
    const source = asOptionalRecord(item.source);
    const target = asOptionalRecord(item.target);
@@ -2785,11 +2832,21 @@ function renderRunOnceHuman(payload: Record<string, unknown>): string {
  });
  const next = asOptionalRecord(payload.next);
  const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48);
+  const writeRows = stateWrites.map((item) => [
+    item.follower,
+    item.ok === true ? "ok" : "failed",
+    item.beforeResourceVersion ?? "-",
+    item.afterResourceVersion ?? "-",
+    item.preservedTiming === true ? "yes" : "no",
+    item.exitCode ?? "-",
+    item.message ?? "-",
+  ]);
  return [
    `CI/CD BRANCH-FOLLOWER RUN-ONCE (${payload.ok === false ? "blocked" : payload.dryRun === true ? "dry-run" : "ok"})`,
    "",
    table(["FOLLOWER", "PHASE", "OBSERVED", "TARGET", "TRIGGERED", "IN_FLIGHT", "DECISION"], rows),
    timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`,
+    writeRows.length === 0 ? "" : `\nSTATE WRITES\n${table(["FOLLOWER", "STATUS", "BEFORE_RV", "AFTER_RV", "PRESERVED", "EXIT", "MESSAGE"], writeRows)}`,
    "",
    "NEXT",
    `status: ${next?.status ?? "-"}`,