// SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower. // Responsibility: YAML-first K8s branch-follower controller, status, and adapter orchestration. import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { isAbsolute } from "node:path"; import { repoRoot, rootPath, type UniDeskConfig } from "./config"; import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; import type { RenderedCliResult } from "./output"; import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; import { agentRunImageArtifact, renderAgentRunGitopsFiles } from "./agentrun-manifests"; import { agentRunPipelineRunName, resolveAgentRunLaneTarget } from "./agentrun-lanes"; import { yamlLaneGitMirrorJobManifest, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLanePipelineRunManifest } from "./agentrun/secrets"; import { yamlLaneK3sBuildImageJobManifest } from "./agentrun/yaml-lane"; import { nodeRuntimePipelineRunName } from "./hwlab-node/cleanup"; import { nodeRuntimeGitMirrorJobManifest } from "./hwlab-node/render"; import { nodeRuntimeGitMirrorTarget, nodeRuntimePipelineRunManifest } from "./hwlab-node/web-probe"; import { loadSentinelCicdState } from "./hwlab-node-web-sentinel-cicd"; import { sentinelPublishPipelineRunManifest } from "./hwlab-node-web-sentinel-cicd-jobs"; import { sentinelPipelineRunName } from "./hwlab-node-web-sentinel-cicd-shared"; import { transPath } from "./hwlab-node/runtime-common"; import { configRefGraph, resolveConfigRefString } from "./ops/config-refs"; import { renderControllerManifests, renderControllerReconcileJob, waitForJobShell } from "./cicd-controller-render"; import { buildDebugStep, renderDebugStepHuman } from "./cicd-debug"; import { runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh"; import { nativeCicdScriptLoadShell, readNativeObjectBundle } from "./cicd-native-bundle"; import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native"; import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types"; import { arrayField, asRecord, booleanField, integerField, readYamlRecord, recordField, redactText, shQuote, stringArrayField, stringField, } from "./platform-infra-ops-library"; const DEFAULT_CONFIG_PATH = "config/cicd-branch-followers.yaml"; const SPEC_REF = "PJ2026-01060703"; const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower"; export function cicdHelp(): unknown { return { command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs", output: "text by default; use --json, --raw, or -o json|yaml for machine output", usage: [ "bun scripts/cli.ts cicd branch-follower plan", "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", "bun scripts/cli.ts cicd branch-follower status", "bun scripts/cli.ts cicd branch-follower status --live", "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", "bun scripts/cli.ts cicd branch-follower run-once --follower hwlab-jd01-v03 --confirm --wait", "bun scripts/cli.ts cicd branch-follower debug-step --follower web-probe-sentinel-master --step state-read", "bun scripts/cli.ts cicd branch-follower debug-step --follower web-probe-sentinel-master --step state-write --confirm", "bun scripts/cli.ts cicd branch-follower cleanup-state --follower web-probe-sentinel-master --confirm", "bun scripts/cli.ts cicd branch-follower events --follower agentrun-jd01-v02", "bun scripts/cli.ts cicd branch-follower logs --follower web-probe-sentinel-master", ], config: DEFAULT_CONFIG_PATH, spec: `${SPEC_REF} ${SPEC_VERSION}`, description: "Deploy and inspect the YAML-first Kubernetes branch follower that follows HWLAB v0.3, AgentRun v0.2, and the selected web-probe sentinel master lane without using host worktrees as source authority.", }; } export async function runCicdCommand(_config: UniDeskConfig | null, args: string[]): Promise { const top = args[0]; if (top === undefined || isHelpToken(top)) return renderMachine("cicd", cicdHelp(), "json"); if (top !== "branch-follower") { throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs"); } const options = parseOptions(args.slice(1)); const command = commandLabel(options); if (options.action === "help") return renderMachine(command, cicdHelp(), "json"); const registry = readRegistry(options.configPath); switch (options.action) { case "plan": return renderResult(command, buildPlan(registry, options), options); case "apply": return renderResult(command, await applyController(registry, options), options); case "status": return renderResult(command, await buildStatus(registry, options), options); case "run-once": return renderResult(command, await runOnce(registry, options), options); case "debug-step": return renderResult(command, await buildDebugStep(registry, options, { selectFollowers, readK8sState, readAdapterStatus, decideAndMaybeTrigger, writeFollowerState, runKubeScript }), options); case "cleanup-state": return renderResult(command, cleanupState(registry, options), options); case "events": case "logs": return renderResult(command, await runFollowerDrillDown(registry, options), options); case "help": return renderMachine(command, cicdHelp(), "json"); } } function parseOptions(args: string[]): ParsedOptions { const actionToken = args[0]; if (actionToken === undefined || isHelpToken(actionToken)) { return defaultOptions("help", args.slice(actionToken === undefined ? 0 : 1)); } if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs"].includes(actionToken)) { throw new Error(`cicd branch-follower unknown action: ${actionToken}`); } const action = actionToken as BranchFollowerAction; const options = defaultOptions(action, []); const rest = args.slice(1); for (let index = 0; index < rest.length; index += 1) { const arg = rest[index] ?? ""; if (isHelpToken(arg)) { options.action = "help"; } else if (arg === "--config") { options.configPath = valueOption(rest, ++index, arg); } else if (arg === "--follower" || arg === "--target") { options.followerId = simpleId(valueOption(rest, ++index, arg), arg); } else if (arg === "--all") { options.all = true; } else if (arg === "--confirm") { options.confirm = true; } else if (arg === "--dry-run") { options.dryRun = true; } else if (arg === "--wait") { options.wait = true; } else if (arg === "--in-cluster") { options.inCluster = true; } else if (arg === "--controller") { if (isInClusterRuntime()) options.inCluster = true; else if (options.action === "status") options.live = true; } else if (arg === "--live") { options.live = true; } else if (arg === "--no-live") { options.noLive = true; } else if (arg === "--full") { options.full = true; } else if (arg === "--raw" || arg === "--json") { options.raw = true; options.full = true; options.output = "json"; } else if (arg === "--record-state") { options.recordState = true; } else if (arg === "--step") { options.debugStep = debugStepOption(valueOption(rest, ++index, arg)); } else if (arg === "-o" || arg === "--output") { const value = valueOption(rest, ++index, arg); if (value !== "json" && value !== "yaml" && value !== "wide" && value !== "text") throw new Error(`${arg} must be json, yaml, wide, or text`); options.output = value === "wide" || value === "text" ? "human" : value; if (value === "json" || value === "yaml") { options.raw = true; options.full = true; } } else if (arg === "--limit") { options.limit = positiveInt(valueOption(rest, ++index, arg), arg); } else if (arg === "--tail-bytes" || arg === "--tail") { options.tailBytes = positiveInt(valueOption(rest, ++index, arg), arg); } else if (arg === "--timeout-seconds") { options.timeoutSeconds = positiveInt(valueOption(rest, ++index, arg), arg); } else { throw new Error(`unsupported cicd branch-follower option: ${arg}`); } } if (options.confirm && options.dryRun) throw new Error("cicd branch-follower accepts only one of --confirm or --dry-run"); if (options.action === "apply" && !options.confirm) options.dryRun = true; if (options.action === "run-once" && !options.confirm) options.dryRun = true; if (options.action === "debug-step" && !options.confirm) options.dryRun = true; if (options.action === "cleanup-state" && !options.confirm) options.dryRun = true; if (options.action === "run-once" && options.confirm && !options.all && options.followerId === null) { throw new Error("run-once --confirm requires --all or --follower "); } if (options.action === "cleanup-state" && options.confirm && !options.all && options.followerId === null) { throw new Error("cleanup-state --confirm requires --all or --follower "); } if (options.action === "debug-step" && options.followerId === null) { throw new Error("debug-step requires --follower "); } return options; } function debugStepOption(value: string): BranchFollowerDebugStep { if (value === "state-read" || value === "status-read" || value === "decide" || value === "state-write") return value; throw new Error("--step must be state-read, status-read, decide, or state-write"); } function isInClusterRuntime(): boolean { return Boolean(process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT); } function defaultOptions(action: BranchFollowerAction, _args: string[]): ParsedOptions { return { action, configPath: DEFAULT_CONFIG_PATH, followerId: null, all: false, confirm: false, dryRun: false, wait: false, inCluster: false, live: false, noLive: false, full: false, raw: false, recordState: false, debugStep: null, output: "human", limit: 20, tailBytes: 12000, timeoutSeconds: null, }; } function valueOption(args: string[], index: number, option: string): string { const value = args[index]; if (value === undefined || value.length === 0 || value.startsWith("--")) throw new Error(`${option} requires a value`); return value; } function simpleId(value: string, option: string): string { if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${option} must be a simple id`); return value; } function positiveInt(value: string, option: string): number { const parsed = Number(value); if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${option} must be a positive integer`); return parsed; } function isHelpToken(value: string): boolean { return value === "-h" || value === "--help" || value === "help"; } function readRegistry(configPath: string): BranchFollowerRegistry { const absolute = isAbsolute(configPath) ? configPath : rootPath(configPath); const rawText = readFileSync(absolute, "utf8"); const root = readYamlRecord>(absolute, "CicdBranchFollowerRegistry"); const metadata = recordField(root, "metadata", configPath); const controller = parseController(recordField(root, "controller", configPath)); const followers = arrayField(root, "followers", configPath).map(parseFollower); const ids = new Set(); for (const follower of followers) { if (ids.has(follower.id)) throw new Error(`${configPath}.followers has duplicate id ${follower.id}`); ids.add(follower.id); } return { path: configPath, rawText, rawSha256: createHash("sha256").update(rawText).digest("hex"), metadata: { id: stringField(metadata, "id", `${configPath}.metadata`), owner: stringField(metadata, "owner", `${configPath}.metadata`), specRef: stringField(metadata, "specRef", `${configPath}.metadata`), version: stringField(metadata, "version", `${configPath}.metadata`), }, controller, followers, }; } function parseController(root: Record): ControllerSpec { const source = recordField(root, "source", "controller"); const authority = recordField(source, "sourceAuthority", "controller.source"); const snapshot = recordField(source, "sourceSnapshot", "controller.source"); const loop = recordField(root, "loop", "controller"); const budgets = recordField(root, "budgets", "controller"); const result: ControllerSpec = { namespace: stringField(root, "namespace", "controller"), kubeRoute: stringField(root, "kubeRoute", "controller"), fieldManager: stringField(root, "fieldManager", "controller"), serviceAccountName: stringField(root, "serviceAccountName", "controller"), deploymentName: stringField(root, "deploymentName", "controller"), configMapName: stringField(root, "configMapName", "controller"), stateConfigMapName: stringField(root, "stateConfigMapName", "controller"), leaseName: stringField(root, "leaseName", "controller"), image: stringField(root, "image", "controller"), labels: stringMap(recordField(root, "labels", "controller"), "controller.labels"), source: { repository: stringField(source, "repository", "controller.source"), branch: stringField(source, "branch", "controller.source"), gitMirrorReadUrl: stringField(source, "gitMirrorReadUrl", "controller.source"), gitMirrorCachePvcName: stringField(source, "gitMirrorCachePvcName", "controller.source"), githubSsh: parseControllerGithubSsh(recordField(source, "githubSsh", "controller.source")), sourceAuthority: { mode: stringField(authority, "mode", "controller.source.sourceAuthority"), resolver: stringField(authority, "resolver", "controller.source.sourceAuthority"), allowHostGit: booleanField(authority, "allowHostGit", "controller.source.sourceAuthority"), allowHostWorkspace: booleanField(authority, "allowHostWorkspace", "controller.source.sourceAuthority"), allowGithubDirectInPipeline: booleanField(authority, "allowGithubDirectInPipeline", "controller.source.sourceAuthority"), }, sourceSnapshot: { stageRefPrefix: stringField(snapshot, "stageRefPrefix", "controller.source.sourceSnapshot"), missingObjectPolicy: stringField(snapshot, "missingObjectPolicy", "controller.source.sourceSnapshot"), refreshPolicy: stringField(snapshot, "refreshPolicy", "controller.source.sourceSnapshot"), }, }, loop: { intervalSeconds: integerField(loop, "intervalSeconds", "controller.loop"), reconcileTimeoutSeconds: integerField(loop, "reconcileTimeoutSeconds", "controller.loop"), leaseDurationSeconds: integerField(loop, "leaseDurationSeconds", "controller.loop"), terminationGracePeriodSeconds: integerField(loop, "terminationGracePeriodSeconds", "controller.loop"), }, budgets: { applyWaitSeconds: integerField(budgets, "applyWaitSeconds", "controller.budgets"), statusSeconds: integerField(budgets, "statusSeconds", "controller.budgets"), runOnceSeconds: integerField(budgets, "runOnceSeconds", "controller.budgets"), reconcileJobTtlSeconds: integerField(budgets, "reconcileJobTtlSeconds", "controller.budgets"), reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"), reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"), reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"), nativeTransportGraceSeconds: integerField(budgets, "nativeTransportGraceSeconds", "controller.budgets"), nativePollIntervalSeconds: integerField(budgets, "nativePollIntervalSeconds", "controller.budgets"), }, }; if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) { throw new Error("controller.source.sourceAuthority must disable host git, host workspace, and direct GitHub pipeline fallback"); } return result; } function parseControllerGithubSsh(root: Record): ControllerSpec["source"]["githubSsh"] { return { secretName: stringField(root, "secretName", "controller.source.githubSsh"), privateKeySecretKey: stringField(root, "privateKeySecretKey", "controller.source.githubSsh"), proxyHost: stringField(root, "proxyHost", "controller.source.githubSsh"), proxyPort: integerField(root, "proxyPort", "controller.source.githubSsh"), }; } function parseFollower(root: Record, index: number): FollowerSpec { const label = `followers[${index}]`; const source = recordField(root, "source", label); const target = recordField(root, "target", label); const budgets = recordField(root, "budgets", label); const commands = recordField(root, "commands", label); const nativeStatus = recordField(root, "nativeStatus", label); const closeout = recordField(root, "closeout", label); const configRefs = stringMap(recordField(target, "configRefs", `${label}.target`), `${label}.target.configRefs`); return { id: simpleId(stringField(root, "id", label), `${label}.id`), enabled: booleanField(root, "enabled", label), adapter: stringField(root, "adapter", label), description: stringField(root, "description", label), source: { repository: stringField(source, "repository", `${label}.source`), branch: stringField(source, "branch", `${label}.source`), branchRef: stringField(source, "branchRef", `${label}.source`), authorityRef: stringField(source, "authorityRef", `${label}.source`), snapshotPrefix: stringField(source, "snapshotPrefix", `${label}.source`), snapshotRef: stringField(source, "snapshotRef", `${label}.source`), }, target: { node: stringField(target, "node", `${label}.target`), lane: stringField(target, "lane", `${label}.target`), namespace: stringField(target, "namespace", `${label}.target`), sentinel: typeof target.sentinel === "string" && target.sentinel.length > 0 ? target.sentinel : null, configRefs, }, budgets: { endToEndSeconds: integerField(budgets, "endToEndSeconds", `${label}.budgets`), statusSeconds: integerField(budgets, "statusSeconds", `${label}.budgets`), triggerSeconds: integerField(budgets, "triggerSeconds", `${label}.budgets`), sourceSyncSeconds: integerField(budgets, "sourceSyncSeconds", `${label}.budgets`), controlPlaneRefreshSeconds: integerField(budgets, "controlPlaneRefreshSeconds", `${label}.budgets`), capabilityJobTtlSeconds: integerField(budgets, "capabilityJobTtlSeconds", `${label}.budgets`), capabilityJobBackoffLimit: integerField(budgets, "capabilityJobBackoffLimit", `${label}.budgets`), }, commands: { plan: parseCommand(recordField(commands, "plan", `${label}.commands`), `${label}.commands.plan`), status: parseCommand(recordField(commands, "status", `${label}.commands`), `${label}.commands.status`), trigger: parseCommand(recordField(commands, "trigger", `${label}.commands`), `${label}.commands.trigger`), events: parseCommand(recordField(commands, "events", `${label}.commands`), `${label}.commands.events`), logs: parseCommand(recordField(commands, "logs", `${label}.commands`), `${label}.commands.logs`), }, nativeStatus: parseNativeStatus(nativeStatus, `${label}.nativeStatus`), closeoutChecks: stringArrayField(closeout, "checks", `${label}.closeout`), }; } function parseNativeStatus(root: Record, label: string): NativeStatusSpec { const source = recordField(root, "source", label); const tekton = asOptionalRecord(root.tekton); const argo = asOptionalRecord(root.argo); const runtime = asOptionalRecord(root.runtime); return { source: { gitMirrorReadUrl: stringField(source, "gitMirrorReadUrl", `${label}.source`), gitMirrorNamespace: stringField(source, "gitMirrorNamespace", `${label}.source`), gitMirrorDeployment: stringField(source, "gitMirrorDeployment", `${label}.source`), repoPath: stringField(source, "repoPath", `${label}.source`), }, tekton: tekton === null ? null : { namespace: stringField(tekton, "namespace", `${label}.tekton`), pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${label}.tekton`), }, argo: argo === null ? null : { namespace: stringField(argo, "namespace", `${label}.argo`), application: stringField(argo, "application", `${label}.argo`), }, runtime: runtime === null ? null : { namespace: stringField(runtime, "namespace", `${label}.runtime`), workloads: arrayField(runtime, "workloads", `${label}.runtime`).map((item, index) => parseNativeWorkload(item, `${label}.runtime.workloads[${index}]`)), }, }; } function parseNativeWorkload(value: unknown, label: string): NativeWorkloadSpec { const root = asRecord(value, label); const kind = stringField(root, "kind", label); if (kind !== "Deployment" && kind !== "StatefulSet") throw new Error(`${label}.kind must be Deployment or StatefulSet`); const sourceCommit = asOptionalRecord(root.sourceCommit) ?? {}; return { kind, name: stringField(root, "name", label), sourceCommit: { labels: optionalStringArrayField(sourceCommit, "labels", `${label}.sourceCommit`), annotations: optionalStringArrayField(sourceCommit, "annotations", `${label}.sourceCommit`), podLabels: optionalStringArrayField(sourceCommit, "podLabels", `${label}.sourceCommit`), podAnnotations: optionalStringArrayField(sourceCommit, "podAnnotations", `${label}.sourceCommit`), env: optionalStringArrayField(sourceCommit, "env", `${label}.sourceCommit`), }, }; } function parseCommand(root: Record, label: string): CommandSpec { return { argv: stringArrayField(root, "argv", label), timeoutSeconds: integerField(root, "timeoutSeconds", label), }; } function optionalStringArrayField(root: Record, key: string, label: string): string[] { return root[key] === undefined ? [] : stringArrayField(root, key, label); } function stringMap(root: Record, label: string): Record { const result: Record = {}; for (const [key, value] of Object.entries(root)) { if (typeof value !== "string" || value.length === 0) throw new Error(`${label}.${key} must be a non-empty string`); result[key] = value; } return result; } function buildPlan(registry: BranchFollowerRegistry, options: ParsedOptions): Record { const selected = selectFollowers(registry, options, { includeDisabled: true }); const followers = selected.map((follower) => { const branchValue = safeResolveString(follower.source.branchRef); const graph = configRefGraph([ { id: "source.branch", ref: follower.source.branchRef }, { id: "source.authority", ref: follower.source.authorityRef }, { id: "source.snapshot", ref: follower.source.snapshotRef }, ...Object.entries(follower.target.configRefs).map(([id, ref]) => ({ id: `target.${id}`, ref })), ]); const warnings: string[] = []; if (branchValue !== null && branchValue !== follower.source.branch) warnings.push(`source.branch ${follower.source.branch} differs from ${follower.source.branchRef} value ${branchValue}`); return { id: follower.id, enabled: follower.enabled, adapter: follower.adapter, description: follower.description, source: { repository: follower.source.repository, branch: follower.source.branch, branchRef: follower.source.branchRef, resolvedBranch: branchValue, snapshotPrefix: follower.source.snapshotPrefix, }, target: follower.target, budgets: follower.budgets, commands: redactCommands(follower), nativeStatus: nativeStatusPlan(follower.nativeStatus), closeoutChecks: follower.closeoutChecks, configRefGraph: graph, warnings, }; }); return { ok: true, action: "plan", spec: `${SPEC_REF} ${SPEC_VERSION}`, registry: registrySummary(registry), hostWorktreeAuthority: false, sourceAuthority: { mode: registry.controller.source.sourceAuthority.mode, resolver: registry.controller.source.sourceAuthority.resolver, allowHostGit: registry.controller.source.sourceAuthority.allowHostGit, allowHostWorkspace: registry.controller.source.sourceAuthority.allowHostWorkspace, allowGithubDirectInPipeline: registry.controller.source.sourceAuthority.allowGithubDirectInPipeline, }, controller: { namespace: registry.controller.namespace, kubeRoute: registry.controller.kubeRoute, deploymentName: registry.controller.deploymentName, stateConfigMapName: registry.controller.stateConfigMapName, leaseName: registry.controller.leaseName, image: registry.controller.image, loop: registry.controller.loop, budgets: registry.controller.budgets, }, followers, next: { apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", status: "bun scripts/cli.ts cicd branch-follower status", dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", }, }; } async function applyController(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { const manifests = renderControllerManifests(registry); const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; const manifestBase64 = Buffer.from(manifestYaml, "utf8").toString("base64"); const waitSeconds = options.timeoutSeconds ?? registry.controller.budgets.applyWaitSeconds; const script = [ "set -eu", "tmp=$(mktemp)", "base64 -d >\"$tmp\" <<'UNIDESK_CICD_BRANCH_FOLLOWER_MANIFEST_B64'", manifestBase64, "UNIDESK_CICD_BRANCH_FOLLOWER_MANIFEST_B64", options.dryRun ? `kubectl apply --dry-run=server --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp"` : `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp"`, !options.dryRun && options.wait ? `kubectl -n ${shQuote(registry.controller.namespace)} rollout status deploy/${shQuote(registry.controller.deploymentName)} --timeout=${waitSeconds}s` : "true", `kubectl -n ${shQuote(registry.controller.namespace)} get deploy/${shQuote(registry.controller.deploymentName)} cm/${shQuote(registry.controller.configMapName)} cm/${shQuote(registry.controller.stateConfigMapName)} lease/${shQuote(registry.controller.leaseName)} -o wide 2>/dev/null || true`, ].join("\n"); const result = runKubeScript(registry, options, script, "", (waitSeconds + 15) * 1000); return { ok: result.exitCode === 0, action: "apply", dryRun: options.dryRun, wait: options.wait, registry: registrySummary(registry), objects: manifests.map((item) => objectRef(item)), manifestSha256: createHash("sha256").update(manifestYaml).digest("hex"), controller: { namespace: registry.controller.namespace, route: registry.controller.kubeRoute, deploymentName: registry.controller.deploymentName, stateConfigMapName: registry.controller.stateConfigMapName, leaseName: registry.controller.leaseName, hostWorktreeMounted: false, sourceMode: "k8s-git-mirror-to-emptyDir", }, command: commandCompact(result, options), next: { status: "bun scripts/cli.ts cicd branch-follower status", logs: `bun scripts/cli.ts cicd branch-follower logs --follower ${registry.followers[0]?.id ?? ""}`, dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", }, }; } async function buildStatus(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { let k8s = readK8sState(registry, options); const wantsLive = options.live || (!options.noLive && Object.keys(k8s.stateByFollower).length === 0); const refresh = wantsLive && !options.inCluster ? runControllerReconcileJob(registry, options, { dryRun: true, wait: true, recordState: true }) : null; if (refresh !== null) k8s = readK8sState(registry, options); const shouldLive = wantsLive && options.inCluster; const selected = selectFollowers(registry, options, { includeDisabled: true }); const followers = []; for (const follower of selected) { const stored = k8s.stateByFollower[follower.id] ?? {}; const live = shouldLive && follower.enabled ? await readAdapterStatus(registry, follower, options) : null; followers.push(mergeFollowerStatus(registry, follower, stored, live, shouldLive)); } return { ok: k8s.ok && followers.every((item) => item.ok !== false), action: "status", live: shouldLive, registry: registrySummary(registry), controller: controllerStatusSummary(registry, k8s), followers, refresh, errors: k8s.errors, next: { apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", liveStatus: "bun scripts/cli.ts cicd branch-follower status --live", dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", }, }; } async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { if (!options.inCluster) { const refresh = runControllerReconcileJob(registry, options, { dryRun: options.dryRun, wait: true, recordState: true }); const k8s = readK8sState(registry, options); const selected = selectFollowers(registry, options, { includeDisabled: false }); return { ok: refresh.ok, action: "run-once", dryRun: options.dryRun, confirm: options.confirm, wait: true, controller: false, execution: "k8s-native-reconcile-job", registry: registrySummary(registry), job: refresh, followers: selected.map((follower) => mergeFollowerStatus(registry, follower, k8s.stateByFollower[follower.id] ?? {}, null, false)), warnings: refresh.ok ? [] : [`reconcile job failed: ${refresh.message}`], next: { status: "bun scripts/cli.ts cicd branch-follower status", liveStatus: "bun scripts/cli.ts cicd branch-follower status --live", }, }; } const selected = selectFollowers(registry, options, { includeDisabled: false }); const previous = readK8sState(registry, options); const results: FollowerState[] = []; const stateWriteWarnings: string[] = []; const stateWrites: Record[] = []; for (const follower of selected) { const oldState = previous.stateByFollower[follower.id] ?? {}; const live = await readAdapterStatus(registry, follower, options); const state = await decideAndMaybeTrigger(registry, follower, oldState, live, options); if (!options.dryRun || options.recordState) { const write = writeFollowerState(registry, state, options); const writeSummary = stateWriteSummary(follower.id, write); stateWrites.push(writeSummary); if (write.exitCode !== 0) { const warning = `state write failed for ${follower.id}: ${tailText(write.stderr || write.stdout, 300)}`; state.warnings.push(warning); stateWriteWarnings.push(warning); } } results.push(state); } return { ok: results.every((item) => item.phase !== "Failed" && item.phase !== "Blocked"), action: "run-once", dryRun: options.dryRun, confirm: options.confirm, wait: options.wait, controller: options.inCluster, registry: registrySummary(registry), followers: results, stateWrites, warnings: stateWriteWarnings, next: { status: "bun scripts/cli.ts cicd branch-follower status", liveStatus: "bun scripts/cli.ts cicd branch-follower status --live", }, }; } function cleanupState(registry: BranchFollowerRegistry, options: ParsedOptions): Record { const selected = selectFollowers(registry, options, { includeDisabled: true }); const ids = selected.map((follower) => follower.id); const before = kubeConfigMapFollowerState(registry, options); const presentKeys = ids.filter((id) => before.stateByFollower[id] !== undefined); const command = options.confirm ? removeFollowerStateKeys(registry, options, ids) : null; return { ok: command === null ? before.ok : command.exitCode === 0, action: "cleanup-state", dryRun: !options.confirm, confirm: options.confirm, execution: "k8s-state-configmap-key-delete", registry: registrySummary(registry), controller: { namespace: registry.controller.namespace, route: registry.controller.kubeRoute, stateConfigMapName: registry.controller.stateConfigMapName, }, followers: ids.map((id) => ({ id, statePresent: before.stateByFollower[id] !== undefined, cleanup: options.confirm ? "deleted-if-present" : "would-delete-if-present", })), stateConfigMapPresent: before.present, presentKeys, parsedDownstreamCliOutput: false, command: command === null ? null : commandCompact(command, options), errors: before.ok ? [] : [before.error], next: { status: "bun scripts/cli.ts cicd branch-follower status", runOnce: options.followerId === null ? "bun scripts/cli.ts cicd branch-follower run-once --all --confirm --wait" : `bun scripts/cli.ts cicd branch-follower run-once --follower ${options.followerId} --confirm --wait`, }, }; } async function runFollowerDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { if (options.followerId === null) { return { ok: true, action: options.action, message: "select one follower to inspect native Kubernetes status objects", followers: registry.followers.map((follower) => ({ id: follower.id, adapter: follower.adapter, statusAuthority: "k8s-native", nativeStatus: nativeStatusPlan(follower.nativeStatus), })), }; } const follower = registry.followers.find((item) => item.id === options.followerId); if (follower === undefined) throw new Error(`unknown follower ${options.followerId}`); if (!options.inCluster) { const refresh = runControllerReconcileJob(registry, options, { dryRun: true, wait: true, recordState: true }); const k8s = readK8sState(registry, options); const stored = k8s.stateByFollower[follower.id] ?? {}; const storedSource = asOptionalRecord(stored.source); const storedTarget = asOptionalRecord(stored.target); const command = asOptionalRecord(stored.command); const native = asOptionalRecord(command?.payload); return { ok: refresh.ok && k8s.ok && Object.keys(stored).length > 0, action: options.action, follower: follower.id, adapter: follower.adapter, statusAuthority: "k8s-native-state-configmap", parsedDownstreamCliOutput: false, summary: { phase: stringOrNull(stored.phase) ?? "Observed", observedSha: stringOrNull(storedSource?.observedSha), targetSha: stringOrNull(storedTarget?.targetSha), pipelineRun: stringOrNull(stored.pipelineRun) ?? stringOrNull(stored.inFlightJob), aligned: null, message: stringOrNull(stored.decision) ?? "no controller state yet", }, native, refresh, errors: k8s.errors, next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, liveStatus: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --live`, runOnceDryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`, }, }; } const live = await readAdapterStatus(registry, follower, options); return { ok: live.ok, action: options.action, follower: follower.id, adapter: follower.adapter, statusAuthority: "k8s-native", parsedDownstreamCliOutput: false, summary: { phase: live.phase, observedSha: live.observedSha, targetSha: live.targetSha, pipelineRun: live.pipelineRun, aligned: live.aligned, message: live.message, }, native: live.payload, next: { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, runOnceDryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`, }, }; } async function decideAndMaybeTrigger( registry: BranchFollowerRegistry, follower: FollowerSpec, previous: Record, live: AdapterSummary, options: ParsedOptions, ): Promise { const warnings: string[] = []; if (!live.ok) warnings.push(`status command failed: exitCode=${live.exitCode}${live.timedOut ? " timedOut=true" : ""}`); const observedSha = live.observedSha; let targetSha = live.targetSha; const previousPhase = stringOrNull(previous.phase); const previousLastTriggered = previousPhase === "Failed" ? null : stringOrNull(previous.lastTriggeredSha); const previousInFlight = stringOrNull(previous.inFlightJob); const previousLastSucceeded = stringOrNull(previous.lastSucceededSha); const previousObserved = stringOrNull(recordAt(previous, ["source"])?.observedSha); const previousTarget = stringOrNull(recordAt(previous, ["target"])?.targetSha); const superseded = previousInFlight !== null && previousObserved !== null && observedSha !== null && previousObserved !== observedSha; const nativePipelineRunMatchesObserved = observedSha !== null && live.pipelineRunPresent === true && live.pipelineRun !== null && live.pipelineRun === expectedPipelineRunName(follower, observedSha); let phase: BranchFollowerPhase; let decision: string; let triggerCommand: Record | undefined; let inFlightJob: string | null = live.inFlightJob; let lastTriggeredSha = live.lastTriggeredSha ?? previousLastTriggered ?? (nativePipelineRunMatchesObserved ? observedSha : null); let lastSucceededSha = live.lastSucceededSha ?? previousLastSucceeded; if (targetSha === null && observedSha !== null && previousLastSucceeded === observedSha && previousTarget === observedSha) targetSha = observedSha; if (!follower.enabled) { phase = "Skipped"; decision = "follower disabled"; } else if (observedSha === null) { phase = live.ok ? "Observed" : "Blocked"; decision = "status did not expose an observed source sha; adapter trigger-current remains the dedupe authority"; } else if (superseded) { phase = "Superseded"; decision = `previous in-flight sha ${shortSha(previousObserved)} was superseded by ${shortSha(observedSha)}`; } else if (targetSha !== null && targetSha === observedSha && live.aligned === true) { phase = "Noop"; decision = "target already matches observed source sha"; inFlightJob = null; lastSucceededSha = observedSha; } else if (lastTriggeredSha !== null && lastTriggeredSha === observedSha && live.pipelineRunPresent === true) { phase = "ClosingOut"; decision = targetSha === observedSha ? "target sha matches observed source sha but native closeout gates are not complete" : "same sha was already triggered; use status/events/logs for closeout"; } else if (lastTriggeredSha !== null && lastTriggeredSha === observedSha) { phase = "PendingTrigger"; decision = "same sha has a trigger record but no PipelineRun is present; retrying trigger"; } else if (targetSha !== null && targetSha === observedSha) { phase = "ClosingOut"; decision = "target sha matches observed source sha but native closeout gates are not complete"; } else { phase = "PendingTrigger"; decision = targetSha === null ? "target sha is unknown; trigger-current adapter will dedupe by source snapshot" : `observed ${shortSha(observedSha)} differs from target ${shortSha(targetSha)}`; } if (options.confirm && (phase === "PendingTrigger" || phase === "Superseded" || (phase === "Observed" && observedSha !== null))) { const trigger = await executeTrigger(registry, follower, observedSha, options); triggerCommand = trigger.command; phase = trigger.ok ? (options.wait ? "ClosingOut" : "Triggering") : "Failed"; decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}: ${redactText(trigger.message).slice(0, 220)}`; inFlightJob = trigger.jobId ?? live.inFlightJob; lastTriggeredSha = observedSha; if (trigger.ok && options.wait && trigger.completed) { phase = "Succeeded"; decision = `trigger completed for ${shortSha(observedSha)}`; inFlightJob = null; targetSha = observedSha; lastSucceededSha = observedSha; } if (!trigger.ok) warnings.push(trigger.message); } if (options.confirm && options.wait && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) { const closeout = await waitNativeFollowerCloseout(registry, follower, observedSha, options, options.timeoutSeconds ?? follower.budgets.endToEndSeconds); triggerCommand = closeoutOnlyCommand(follower, live.pipelineRun, observedSha, closeout); if (closeout.completed) { phase = "Succeeded"; decision = `closeout completed for ${shortSha(observedSha)}`; inFlightJob = null; targetSha = observedSha; lastSucceededSha = observedSha; } else { decision = closeout.timedOut ? `closeout did not converge for ${shortSha(observedSha)} within budget` : `closeout remains pending for ${shortSha(observedSha)}`; warnings.push(decision); } } if (options.dryRun && phase === "PendingTrigger") decision = `${decision}; dry-run did not trigger`; const statePipelineRun = stringOrNull(triggerCommand?.pipelineRun) ?? live.pipelineRun; return { id: follower.id, adapter: follower.adapter, enabled: follower.enabled, phase, source: { repository: follower.source.repository, branch: follower.source.branch, branchRef: follower.source.branchRef, snapshotPrefix: follower.source.snapshotPrefix, observedSha, }, target: { node: follower.target.node, lane: follower.target.lane, namespace: follower.target.namespace, sentinel: follower.target.sentinel, targetSha, }, lastTriggeredSha, lastSucceededSha, pipelineRun: statePipelineRun, inFlightJob, budgetSource: follower.budgets, controller: { mode: options.inCluster ? "k8s-controller" : "local-cli", stateConfigMap: registry.controller.stateConfigMapName, leaseName: registry.controller.leaseName, }, decision, dryRun: options.dryRun, updatedAt: new Date().toISOString(), timings: buildFollowerTimings(follower, live, triggerCommand, asOptionalRecord(previous.timings), phase), warnings, next: followerNextCommands(follower), command: triggerCommand ?? { status: live.command, exitCode: live.exitCode, timedOut: live.timedOut, payload: live.payload, }, }; } async function executeTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions): Promise { const spec = follower.commands.trigger; const timeoutSeconds = options.timeoutSeconds ?? spec.timeoutSeconds; if (follower.adapter === "hwlab-node-runtime" && options.inCluster) { return await executeNativeHwlabNodeTrigger(registry, follower, observedSha, options, timeoutSeconds); } if (follower.adapter === "agentrun-yaml-lane" && options.inCluster) { return await executeNativeAgentRunTrigger(registry, follower, observedSha, options, timeoutSeconds); } if (follower.adapter === "web-probe-sentinel-cicd" && options.inCluster) { return await executeNativeSentinelTrigger(registry, follower, observedSha, options, timeoutSeconds); } if (!options.wait && !options.inCluster) { const job = startJob(`cicd_branch_follower_${safeJobSegment(follower.id)}`, spec.argv, `Trigger ${follower.id} for observed sha ${observedSha ?? "unknown"}`); return { ok: true, completed: false, message: `started async job ${job.id}`, jobId: job.id, command: { mode: "async-job", argv: spec.argv, jobId: job.id, status: `bun scripts/cli.ts job status ${job.id}`, }, }; } const result = runCommand(spec.argv, repoRoot, { timeoutMs: timeoutSeconds * 1000 }); return { ok: result.exitCode === 0, completed: result.exitCode === 0 && options.wait, message: result.exitCode === 0 ? "trigger command completed" : tailText(result.stderr || result.stdout, 500), jobId: null, command: commandCompact(result, options), }; } async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions, timeoutSeconds: number): Promise { if (observedSha === null) { return nativeTriggerError(follower, "native HWLAB trigger requires observed source sha", "observed-sha-missing"); } if (follower.nativeStatus.tekton === null) { return nativeTriggerError(follower, "native HWLAB trigger requires Tekton nativeStatus", "tekton-native-status-missing"); } const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); const pipelineRun = nodeRuntimePipelineRunName(spec, observedSha); const namespace = follower.nativeStatus.tekton.namespace; const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun); const startedAt = Date.now(); const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds)); if (sync !== null && !sync.result.ok) { return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt); } const refresh = runNativeHwlabControlPlaneRefresh(registry, follower, spec, observedSha, Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.controlPlaneRefreshSeconds), nativeCapabilityJobName(follower.id, "control-plane-refresh", observedSha)); if (!refresh.result.ok) { return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt); } const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets); const payload = parseJsonObject(result.stdout) ?? {}; const pipelineRunCompleted = payload.completed === true; const failed = payload.failed === true || result.exitCode !== 0; const flush = !failed && options.wait && pipelineRunCompleted ? runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds)) : null; if (flush !== null && !flush.result.ok) { return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native git-mirror flush failed", startedAt); } const closeout = !failed && options.wait && pipelineRunCompleted ? await waitNativeFollowerCloseout(registry, follower, observedSha, options, remainingSeconds(startedAt, timeoutSeconds)) : null; return nativeTektonTriggerResult({ follower, observedSha, namespace, pipelineRun, stageRef: `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`, wait: options.wait, result, startedAt, payload: { ...payload, nativeCapabilities: { gitMirrorSync: sync === null ? null : sync.result, controlPlaneRefresh: refresh.result, gitMirrorFlush: flush === null ? null : flush.result, }, }, closeout, successMessage: `native HWLAB PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}`, }); } async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions, timeoutSeconds: number): Promise { if (observedSha === null) { return nativeTriggerError(follower, "native AgentRun trigger requires observed source sha", "observed-sha-missing"); } if (follower.nativeStatus.tekton === null) { return nativeTriggerError(follower, "native AgentRun trigger requires Tekton nativeStatus", "tekton-native-status-missing"); } const { configPath, spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); const startedAt = Date.now(); const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`; const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`; const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds)); if (sync !== null && !sync.result.ok) { return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt); } const buildJob = `${jobPrefix}-build-${observedSha.slice(0, 12)}`.slice(0, 63); const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), spec.deployment.manager.imageBuild.timeoutSeconds), "buildkit", registry.controller.budgets); const buildPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(build.logsTail) ?? "" }); const digest = stringOrNull(buildPayload.digest); const envIdentity = stringOrNull(buildPayload.envIdentity); if (!build.ok || digest === null || envIdentity === null) { return nativeK8sStageFailure(follower, observedSha, "image-build", buildJob, build, buildPayload, "native AgentRun image build failed", startedAt); } const image = agentRunImageArtifact(spec, { sourceCommit: observedSha, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" }); const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: observedSha, image }); const publishJob = `${jobPrefix}-gitops-${observedSha.slice(0, 12)}`.slice(0, 63); const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish", registry.controller.budgets); const publishPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(publish.logsTail) ?? "" }); if (!publish.ok || publishPayload.ok === false || stringOrNull(publishPayload.gitopsCommit) === null) { return nativeK8sStageFailure(follower, observedSha, "gitops-publish", publishJob, publish, publishPayload, "native AgentRun GitOps publish failed", startedAt); } const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds)); if (flush !== null && !flush.result.ok) { return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native AgentRun git-mirror flush failed", startedAt); } const pipelineRun = agentRunPipelineRunName(spec, observedSha); const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets); const tektonPayload = parseJsonObject(tektonResult.stdout) ?? {}; const pipelineRunCompleted = tektonPayload.completed === true; const failed = tektonPayload.failed === true || tektonResult.exitCode !== 0; const closeout = !failed && options.wait && pipelineRunCompleted ? await waitNativeFollowerCloseout(registry, follower, observedSha, options, remainingSeconds(startedAt, timeoutSeconds)) : null; const trigger = nativeTektonTriggerResult({ follower, observedSha, namespace: follower.nativeStatus.tekton.namespace, pipelineRun, stageRef, wait: options.wait, result: tektonResult, startedAt, payload: { ...tektonPayload, agentrun: { configPath, gitMirrorSync: sync === null ? null : { jobName: sync.jobName, payload: sync.result }, imageBuild: { jobName: buildJob, result: build, payload: buildPayload }, gitopsPublish: { jobName: publishJob, result: publish, payload: publishPayload }, gitMirrorFlush: flush === null ? null : { jobName: flush.jobName, payload: flush.result }, }, }, closeout, successMessage: `native AgentRun PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}`, }); return trigger; } function nativeTriggerError(follower: FollowerSpec, message: string, reason: string): TriggerResult { return { ok: false, completed: false, message, jobId: null, command: { mode: "k8s-native-tekton", adapter: follower.adapter, ok: false, reason, parsedDownstreamCliOutput: false }, }; } function nativeK8sStageFailure( follower: FollowerSpec, observedSha: string, phase: string, jobName: string, job: NativeK8sJobResult, payload: Record, message: string, startedAt?: number, ): TriggerResult { const detail = [ message, job.timedOut ? "timedOut=true" : null, job.conditionReason === null ? null : `reason=${job.conditionReason}`, job.conditionMessage === null ? null : `condition=${job.conditionMessage}`, job.logsTail === null ? null : `logs=${tailText(job.logsTail, 500)}`, ].filter((item): item is string => item !== null).join("; "); return { ok: false, completed: false, message: detail, jobId: jobName, command: { mode: "k8s-native-job", adapter: follower.adapter, phase, jobName, sourceCommit: observedSha, ok: false, startedAt: startedAt === undefined ? null : new Date(startedAt).toISOString(), finishedAt: new Date().toISOString(), elapsedMs: startedAt === undefined ? job.elapsedMs : Date.now() - startedAt, payload, job, statusAuthority: "kubernetes-api-serviceaccount", parsedDownstreamCliOutput: false, }, }; } function runNativeGitMirrorStage(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string, action: "sync" | "flush", timeoutSeconds: number, jobKey = observedSha): { jobName: string; namespace: string; result: NativeK8sJobResult } | null { const job = nativeGitMirrorJobForFollower(follower, jobKey, action); if (job === null) return null; const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action, registry.controller.budgets); return { jobName: job.jobName, namespace: job.namespace, result }; } function nativeGitMirrorJobForFollower( follower: FollowerSpec, observedSha: string, action: "sync" | "flush", ): { namespace: string; jobName: string; manifest: Record } | null { const jobName = nativeCapabilityJobName(follower.id, action, observedSha); if (follower.adapter === "hwlab-node-runtime") { const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); const mirror = nodeRuntimeGitMirrorTarget(spec); return { namespace: mirror.namespace, jobName, manifest: nodeRuntimeGitMirrorJobManifest(mirror, action, jobName), }; } if (follower.adapter === "agentrun-yaml-lane") { const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); return { namespace: spec.gitMirror.namespace, jobName, manifest: yamlLaneGitMirrorJobManifest(spec, action, jobName), }; } return null; } function nativeTektonTriggerResult(input: { follower: FollowerSpec; observedSha: string; namespace: string; pipelineRun: string; stageRef: string; wait: boolean; result: CommandResult; startedAt: number; payload: Record; closeout: NativeCloseoutWaitResult | null; successMessage: string; }): TriggerResult { const pipelineRunCompleted = input.payload.completed === true; const failed = input.payload.failed === true || input.result.exitCode !== 0; const stillRunning = input.payload.stillRunning === true || input.payload.timedOutWait === true; const message = failed ? nativeTektonFailureText(input.payload, input.result) : input.closeout?.completed === true ? input.successMessage : input.closeout?.timedOut === true ? `native PipelineRun ${input.pipelineRun} succeeded but runtime closeout did not converge within budget` : pipelineRunCompleted ? `native PipelineRun ${input.pipelineRun} succeeded; runtime closeout remains k8s-native` : stillRunning ? `native PipelineRun ${input.pipelineRun} is still running; query status/events/logs for closeout` : `native PipelineRun ${input.pipelineRun} submitted`; const ok = !failed && (input.closeout === null || input.closeout.completed === true); const finishedAt = failed || input.result.timedOut || input.closeout?.completed === true || input.closeout?.timedOut === true ? new Date().toISOString() : null; return { ok, completed: input.closeout?.completed === true, message, jobId: input.pipelineRun, command: { mode: "k8s-native-tekton", adapter: input.follower.adapter, namespace: input.namespace, pipelineRun: input.pipelineRun, sourceCommit: input.observedSha, sourceStageRef: input.stageRef, wait: input.wait, pipelineRunCompleted, stillRunning, startedAt: new Date(input.startedAt).toISOString(), finishedAt, elapsedMs: Date.now() - input.startedAt, closeout: input.closeout, statusAuthority: "kubernetes-api-serviceaccount", parsedDownstreamCliOutput: false, payload: input.payload, exitCode: input.result.exitCode, timedOut: input.result.timedOut, stderrTail: failed ? redactText(tailText(input.result.stderr, 1000)) : "", }, }; } function nativeTektonFailureText(payload: Record, result: CommandResult): string { return [ stringOrNull(payload.message), stringOrNull(payload.conditionReason), stringOrNull(payload.conditionMessage), payload.failed === true ? "PipelineRun failed" : null, result.timedOut ? "timedOut=true" : null, ].filter((item): item is string => item !== null && item.length > 0).join("; ") || "native PipelineRun failed; query events/logs for structured details"; } function remainingSeconds(startedAt: number, timeoutSeconds: number): number { return Math.max(1, timeoutSeconds - Math.ceil((Date.now() - startedAt) / 1000)); } async function executeNativeSentinelTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions, timeoutSeconds: number): Promise { if (observedSha === null) { return { ok: false, completed: false, message: "native sentinel trigger requires observed source sha", jobId: null, command: { mode: "k8s-native-tekton", ok: false, reason: "observed-sha-missing", parsedDownstreamCliOutput: false }, }; } if (follower.target.sentinel === null) { return { ok: false, completed: false, message: "native sentinel trigger requires target.sentinel", jobId: null, command: { mode: "k8s-native-tekton", ok: false, reason: "target-sentinel-missing", parsedDownstreamCliOutput: false }, }; } const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`; const state = loadSentinelCicdState(spec, follower.target.sentinel, timeoutSeconds, "cached", { commit: observedSha, stageRef, mirrorCommit: observedSha, sourceAuthority: "git-mirror-snapshot", }); const pipelineRun = sentinelPipelineRunName(state, false); const namespace = stringField(recordField(state.cicd, "builder", `${follower.id}.sentinel.cicd`), "namespace", `${follower.id}.sentinel.cicd.builder`); const manifest = sentinelPublishPipelineRunManifest(state, pipelineRun, true); const startedAt = Date.now(); const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds, registry.controller.budgets); const payload = parseJsonObject(result.stdout) ?? {}; const pipelineRunCompleted = payload.completed === true; const failed = payload.failed === true || result.exitCode !== 0; const stillRunning = payload.stillRunning === true || payload.timedOutWait === true; const remainingSeconds = Math.max(1, timeoutSeconds - Math.ceil((Date.now() - startedAt) / 1000)); const closeout = !failed && options.wait && pipelineRunCompleted ? await waitNativeSentinelCloseout(registry, follower, observedSha, options, remainingSeconds) : null; const message = failed ? tailText(result.stderr || stringOrNull(payload.message) || result.stdout, 500) : closeout?.completed === true ? `native sentinel PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}` : closeout?.timedOut === true ? `native sentinel PipelineRun ${pipelineRun} succeeded but runtime closeout did not converge within budget` : pipelineRunCompleted ? `native sentinel PipelineRun ${pipelineRun} succeeded; runtime closeout remains k8s-native` : stillRunning ? `native sentinel PipelineRun ${pipelineRun} is still running; query status/events/logs for closeout` : `native sentinel PipelineRun ${pipelineRun} submitted`; const ok = !failed && (closeout === null || closeout.completed === true); const finishedAt = failed || result.timedOut || closeout?.completed === true || closeout?.timedOut === true ? new Date().toISOString() : null; return { ok, completed: closeout?.completed === true, message, jobId: pipelineRun, command: { mode: "k8s-native-tekton", adapter: follower.adapter, namespace, pipelineRun, sourceCommit: observedSha, sourceStageRef: stageRef, wait: options.wait, pipelineRunCompleted, stillRunning, startedAt: new Date(startedAt).toISOString(), finishedAt, elapsedMs: Date.now() - startedAt, closeout, statusAuthority: "kubernetes-api-serviceaccount", parsedDownstreamCliOutput: false, payload, exitCode: result.exitCode, timedOut: result.timedOut, stderrTail: failed ? redactText(tailText(result.stderr, Math.min(options.tailBytes, 4000))) : "", }, }; } async function waitNativeSentinelCloseout( registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string, options: ParsedOptions, timeoutSeconds: number, ): Promise { const refreshResult = follower.nativeStatus.argo === null ? null : runNativeArgoRefresh(follower.nativeStatus.argo); const startedAt = Date.now(); const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000; let polls = 0; let latest: AdapterSummary | null = null; let gitMirrorFlush: Record | null = null; while (Date.now() <= deadline) { polls += 1; const remainingSeconds = Math.max(1, Math.ceil((deadline - Date.now()) / 1000)); latest = await readAdapterStatus(registry, follower, { ...options, timeoutSeconds: Math.min(10, remainingSeconds) }); const latestPayload = asOptionalRecord(latest.payload); const latestGitMirror = asOptionalRecord(latestPayload?.gitMirror); if (latest.observedSha === observedSha && asOptionalRecord(latestPayload?.tekton)?.succeeded === true && gitMirrorFlush === null && shouldFlushNativeGitMirrorDuringCloseout(follower, latestGitMirror)) { const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", Math.min(remainingSeconds, follower.budgets.sourceSyncSeconds), stringOrNull(latestGitMirror?.localGitops) ?? observedSha); gitMirrorFlush = flush === null ? null : { jobName: flush.jobName, namespace: flush.namespace, result: compactNativeK8sJobResult(flush.result), }; if (flush !== null && !flush.result.ok) { return { ok: false, completed: false, timedOut: flush.result.timedOut, polls, elapsedMs: Date.now() - startedAt, refresh: refreshResult === null ? null : commandCompact(refreshResult, options), gitMirrorFlush, summary: nativeCloseoutSummary(latest), statusAuthority: "k8s-native", parsedDownstreamCliOutput: false, }; } continue; } if (latest.observedSha === observedSha && latest.aligned === true) { return { ok: true, completed: true, timedOut: false, polls, elapsedMs: Date.now() - startedAt, refresh: refreshResult === null ? null : commandCompact(refreshResult, options), gitMirrorFlush, summary: nativeCloseoutSummary(latest), statusAuthority: "k8s-native", parsedDownstreamCliOutput: false, }; } if (Date.now() + 2_000 > deadline) break; runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); } return { ok: false, completed: false, timedOut: true, polls, elapsedMs: Date.now() - startedAt, refresh: refreshResult === null ? null : commandCompact(refreshResult, options), gitMirrorFlush, summary: latest === null ? null : nativeCloseoutSummary(latest), statusAuthority: "k8s-native", parsedDownstreamCliOutput: false, }; } function shouldFlushNativeGitMirrorDuringCloseout(follower: FollowerSpec, gitMirror: Record | null): boolean { if (!nativeGitMirrorRequired(follower) || gitMirror === null) return false; if (gitMirror.sourceSnapshotReady === false) return false; return gitMirror.pendingFlush === true || gitMirror.githubInSync === false; } function compactNativeK8sJobResult(result: NativeK8sJobResult): Record { return { ok: result.ok, completed: result.completed, failed: result.failed, timedOut: result.timedOut, created: result.created, reused: result.reused, jobName: result.jobName, namespace: result.namespace, polls: result.polls, elapsedMs: result.elapsedMs, conditionReason: result.conditionReason, conditionMessage: result.conditionMessage, statusAuthority: result.statusAuthority, parsedDownstreamCliOutput: false, }; } async function waitNativeFollowerCloseout( registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string, options: ParsedOptions, timeoutSeconds: number, ): Promise { return await waitNativeSentinelCloseout(registry, follower, observedSha, options, timeoutSeconds); } function closeoutOnlyCommand(follower: FollowerSpec, pipelineRun: string | null, observedSha: string, closeout: NativeCloseoutWaitResult): Record { return { mode: "k8s-native-closeout", adapter: follower.adapter, pipelineRun, sourceCommit: observedSha, wait: true, closeout, finishedAt: closeout.completed || closeout.timedOut ? new Date().toISOString() : null, elapsedMs: closeout.elapsedMs, exitCode: closeout.completed ? 0 : 1, timedOut: closeout.timedOut, statusAuthority: "k8s-native", parsedDownstreamCliOutput: false, }; } function nativeCloseoutSummary(live: AdapterSummary): Record { const payload = asOptionalRecord(live.payload); return { ok: live.ok, phase: live.phase, observedSha: live.observedSha, targetSha: live.targetSha, aligned: live.aligned, pipelineRun: live.pipelineRun, pipelineRunPresent: live.pipelineRunPresent, message: live.message, gitMirror: asOptionalRecord(payload?.gitMirror), tekton: asOptionalRecord(payload?.tekton), taskRuns: compactTaskRunsPayload(asOptionalRecord(payload?.taskRuns)), planArtifacts: compactPlanArtifactsPayload(asOptionalRecord(payload?.planArtifacts)), argo: asOptionalRecord(payload?.argo), runtime: asOptionalRecord(payload?.runtime), errors: Array.isArray(payload?.errors) ? payload.errors.slice(0, 5) : [], }; } function runNativeArgoRefresh(argo: NonNullable): CommandResult { const patchBase64 = Buffer.from(JSON.stringify({ metadata: { annotations: { "argocd.argoproj.io/refresh": "hard", }, }, }), "utf8").toString("base64"); const script = [ "set -eu", `ARGO_NAMESPACE=${shQuote(argo.namespace)}`, `ARGO_APPLICATION=${shQuote(argo.application)}`, `PATCH_B64=${shQuote(patchBase64)}`, "export ARGO_NAMESPACE ARGO_APPLICATION PATCH_B64", "node <<'NODE_ARGO_REFRESH'", "import { readFileSync } from 'node:fs';", "import https from 'node:https';", "const namespace = process.env.ARGO_NAMESPACE || '';", "const application = process.env.ARGO_APPLICATION || '';", "const patch = Buffer.from(process.env.PATCH_B64 || '', 'base64').toString('utf8');", "const host = process.env.KUBERNETES_SERVICE_HOST;", "const port = Number(process.env.KUBERNETES_SERVICE_PORT || '443');", "const token = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/token', 'utf8').trim();", "const ca = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt');", "const path = `/apis/argoproj.io/v1alpha1/namespaces/${encodeURIComponent(namespace)}/applications/${encodeURIComponent(application)}`;", "const req = https.request({ host, port, path, method: 'PATCH', ca, headers: { authorization: `Bearer ${token}`, 'content-type': 'application/merge-patch+json', 'content-length': Buffer.byteLength(patch) } }, (res) => {", " let body = '';", " res.setEncoding('utf8');", " res.on('data', (chunk) => { body += chunk; });", " res.on('end', () => {", " if ((res.statusCode || 0) < 200 || (res.statusCode || 0) >= 300) { process.stderr.write(body || `kube api PATCH argo application status ${res.statusCode}`); process.exit(1); }", " process.stdout.write(JSON.stringify({ ok: true, namespace, application, refresh: 'hard', statusAuthority: 'kubernetes-api-serviceaccount', parsedDownstreamCliOutput: false, valuesRedacted: true }));", " });", "});", "req.on('error', (error) => { process.stderr.write(error?.message || String(error)); process.exit(1); });", "req.write(patch);", "req.end();", "NODE_ARGO_REFRESH", ].join("\n"); return runCommand(["sh", "-lc", script], repoRoot, { timeoutMs: 10_000 }); } async function readAdapterStatus(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Promise { const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.statusSeconds; const startedAt = Date.now(); const sourceSync = runNativeSourceObservationSync(registry, follower, options, Math.min(timeoutSeconds, follower.budgets.sourceSyncSeconds)); const sourceSyncDetail = sourceSync === null || sourceSync.result.ok ? null : redactText(tailText(sourceSync.result.conditionMessage ?? sourceSync.result.logsTail ?? "unknown", 800)); const sourceSyncError = sourceSyncDetail === null ? null : `native source sync failed: ${sourceSyncDetail}`; const bundle = readNativeObjectBundle(registry, follower, options, remainingSeconds(startedAt, timeoutSeconds), runKubeScript); const observedSha = sourceSyncError === null ? stringOrNull(bundle.source?.commit) : null; const runtimeTargetSha = runtimeTargetShaFromWorkloads(follower.nativeStatus.runtime, bundle.workloads); const pipelineRunName = stringOrNull(asOptionalRecord(bundle.pipelineRun?.metadata)?.name) ?? expectedPipelineRunName(follower, observedSha); const pipelineRunPresent = follower.nativeStatus.tekton === null ? null : bundle.pipelineRun !== null; const pipelineSucceeded = pipelineRunSucceeded(bundle.pipelineRun); const argoReady = follower.nativeStatus.argo === null ? null : argoApplicationReady(bundle.argoApplication); const runtimeReady = follower.nativeStatus.runtime === null ? null : runtimeWorkloadsReady(follower.nativeStatus.runtime, bundle.workloads); const gitMirrorRequired = nativeGitMirrorRequired(follower); const gitMirrorReady = gitMirrorRequired ? nativeGitMirrorReady(bundle.gitMirror) : null; const hasTektonGate = follower.nativeStatus.tekton !== null; const hasRuntimeTarget = runtimeTargetSha !== null; const requiresRuntimeTarget = follower.nativeStatus.runtime !== null; const hasTargetEvidence = hasRuntimeTarget || (!requiresRuntimeTarget && hasTektonGate); const pipelineGateOk = !hasTektonGate || pipelineSucceeded === true || (hasRuntimeTarget && runtimeTargetSha === observedSha && argoReady !== false && runtimeReady !== false); const aligned = observedSha !== null && hasTargetEvidence && (requiresRuntimeTarget ? hasRuntimeTarget && runtimeTargetSha === observedSha : true) && pipelineGateOk && gitMirrorReady !== false && argoReady !== false && runtimeReady !== false; const targetSha = hasRuntimeTarget && runtimeTargetSha === observedSha && aligned ? runtimeTargetSha : runtimeTargetSha; const ok = bundle.ok && sourceSyncError === null; const phase = inferPhase(ok, aligned, observedSha, targetSha, bundle.timedOut); const errors = sourceSyncError === null ? bundle.errors : [sourceSyncError, ...bundle.errors]; return { ok, command: "native:k8s-git-mirror+tekton+argocd+runtime", exitCode: bundle.exitCode, timedOut: bundle.timedOut, observedSha, targetSha, lastTriggeredSha: null, lastSucceededSha: aligned && observedSha !== null ? observedSha : null, pipelineRun: pipelineRunName, pipelineRunPresent, inFlightJob: pipelineRunPresent === true && pipelineSucceeded === null && pipelineRunName !== null ? pipelineRunName : null, aligned, phase, message: nativeStatusMessage(ok, phase, observedSha, targetSha, { pipelineSucceeded, gitMirrorReady, argoReady, runtimeReady, errors, }), payload: { source: bundle.source, sourceSync: sourceSync === null ? null : sourceSync.result, gitMirror: nativeGitMirrorSummary(bundle.gitMirror), tekton: nativePipelineRunSummary(bundle.pipelineRun), taskRuns: bundle.taskRuns, planArtifacts: bundle.planArtifacts, argo: nativeArgoSummary(bundle.argoApplication), runtime: nativeRuntimeSummary(follower.nativeStatus.runtime, bundle.workloads, observedSha), timings: { statusRead: { elapsedMs: bundle.elapsedMs, budgetSeconds: timeoutSeconds } }, errors, statusAuthority: "k8s-native", parsedDownstreamCliOutput: false, }, stderrTail: bundle.stderrTail, stdoutTail: bundle.stdoutTail, }; } function runNativeSourceObservationSync(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, timeoutSeconds: number): { jobName: string; namespace: string; result: NativeK8sJobResult } | null { if (!options.inCluster) return null; if (follower.adapter !== "hwlab-node-runtime" && follower.adapter !== "agentrun-yaml-lane") return null; const bucket = Math.floor(Date.now() / (registry.controller.loop.intervalSeconds * 1000)).toString(36); return runNativeGitMirrorStage(registry, follower, "source", "sync", timeoutSeconds, `source-${bucket}`); } function inferPhase(ok: boolean, aligned: boolean | null, observedSha: string | null, targetSha: string | null, timedOut: boolean): BranchFollowerPhase { if (!ok || timedOut) return "Blocked"; if (aligned === true) return "Succeeded"; if (observedSha !== null && targetSha !== null && observedSha === targetSha) return "ClosingOut"; if (observedSha !== null) return "PendingTrigger"; return "Observed"; } function nativeStatusMessage(ok: boolean, phase: BranchFollowerPhase, observedSha: string | null, targetSha: string | null, gates: { pipelineSucceeded: boolean | null; gitMirrorReady: boolean | null; argoReady: boolean | null; runtimeReady: boolean | null; errors: string[] }): string { if (!ok) return gates.errors[0] ?? "native status read failed"; if (phase === "Noop" || phase === "Succeeded") return `target matches ${shortSha(observedSha)}`; if (observedSha !== null) { const gatesText = [ gates.pipelineSucceeded === false ? "pipelineRun not successful" : null, gates.gitMirrorReady === false ? "git-mirror not flushed" : null, gates.argoReady === false ? "argo not healthy/synced" : null, gates.runtimeReady === false ? "runtime not ready" : null, ].filter((item): item is string => item !== null).join("; "); if (targetSha !== null) return gatesText.length > 0 ? `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}; ${gatesText}` : `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}`; return gatesText.length > 0 ? `observed ${shortSha(observedSha)}; ${gatesText}` : `observed ${shortSha(observedSha)} target unknown`; } return "k8s-native status did not expose observed source sha"; } function expectedPipelineRunName(follower: FollowerSpec, observedSha: string | null): string | null { if (observedSha === null || follower.nativeStatus.tekton === null) return null; return `${follower.nativeStatus.tekton.pipelineRunPrefix}-${shortSha(observedSha)}`; } function pipelineRunSucceeded(pipelineRun: Record | null): boolean | null { if (pipelineRun === null) return null; const conditions = Array.isArray(asOptionalRecord(pipelineRun.status)?.conditions) ? asOptionalRecord(pipelineRun.status)?.conditions : []; for (const condition of conditions as unknown[]) { const record = asOptionalRecord(condition); if (record?.type !== "Succeeded") continue; if (record.status === "True") return true; if (record.status === "False") return false; return null; } return null; } function argoApplicationReady(application: Record | null): boolean { if (application === null) return false; const status = asOptionalRecord(application.status); const sync = asOptionalRecord(status?.sync); const health = asOptionalRecord(status?.health); return sync?.status === "Synced" && health?.status === "Healthy"; } function runtimeWorkloadsReady(runtime: NativeStatusSpec["runtime"], workloads: Record[]): boolean { if (runtime === null) return true; if (workloads.length < runtime.workloads.length) return false; return runtime.workloads.every((spec, index) => workloadReady(spec, workloads[index] ?? null)); } function workloadReady(spec: NativeWorkloadSpec, workload: Record | null): boolean { if (workload === null) return false; const status = asOptionalRecord(workload.status); if (spec.kind === "Deployment") { const desired = numberOrNull(asOptionalRecord(workload.spec)?.replicas) ?? 1; const available = numberOrNull(status?.availableReplicas) ?? 0; const updated = numberOrNull(status?.updatedReplicas) ?? 0; return available >= desired && updated >= desired; } const desired = numberOrNull(asOptionalRecord(workload.spec)?.replicas) ?? 1; const ready = numberOrNull(status?.readyReplicas) ?? 0; return ready >= desired; } function runtimeTargetShaFromWorkloads(runtime: NativeStatusSpec["runtime"], workloads: Record[]): string | null { if (runtime === null) return null; const commits: string[] = []; runtime.workloads.forEach((spec, index) => { const commit = workloadSourceCommit(spec, workloads[index] ?? null); if (commit !== null) commits.push(commit); }); const unique = Array.from(new Set(commits)); return unique.length === 1 ? unique[0] ?? null : null; } function workloadSourceCommit(spec: NativeWorkloadSpec, workload: Record | null): string | null { if (workload === null) return null; const metadata = asOptionalRecord(workload.metadata); const labels = asOptionalRecord(metadata?.labels); const annotations = asOptionalRecord(metadata?.annotations); const template = asOptionalRecord(asOptionalRecord(workload.spec)?.template); const podMetadata = asOptionalRecord(template?.metadata); const podLabels = asOptionalRecord(podMetadata?.labels); const podAnnotations = asOptionalRecord(podMetadata?.annotations); for (const key of spec.sourceCommit.labels) { const value = shaOrNull(labels?.[key]); if (value !== null) return value; } for (const key of spec.sourceCommit.annotations) { const value = shaOrNull(annotations?.[key]); if (value !== null) return value; } for (const key of spec.sourceCommit.podLabels) { const value = shaOrNull(podLabels?.[key]); if (value !== null) return value; } for (const key of spec.sourceCommit.podAnnotations) { const value = shaOrNull(podAnnotations?.[key]); if (value !== null) return value; } const containers = Array.isArray(asOptionalRecord(template?.spec)?.containers) ? asOptionalRecord(template?.spec)?.containers : []; for (const envName of spec.sourceCommit.env) { for (const container of containers as unknown[]) { const env = Array.isArray(asOptionalRecord(container)?.env) ? asOptionalRecord(container)?.env : []; for (const entry of env as unknown[]) { const record = asOptionalRecord(entry); if (record?.name !== envName) continue; const value = shaOrNull(record.value); if (value !== null) return value; } } } return null; } function shaOrNull(value: unknown): string | null { return typeof value === "string" && /^[0-9a-f]{40}$/iu.test(value) ? value : null; } function nativePipelineRunSummary(pipelineRun: Record | null): Record | null { if (pipelineRun === null) return null; const metadata = asOptionalRecord(pipelineRun.metadata); const status = asOptionalRecord(pipelineRun.status); const condition = latestCondition(status, "Succeeded"); return { name: stringOrNull(metadata?.name), namespace: stringOrNull(metadata?.namespace), succeeded: pipelineRunSucceeded(pipelineRun), reason: stringOrNull(condition?.reason), startTime: stringOrNull(status?.startTime), completionTime: stringOrNull(status?.completionTime), durationSeconds: numberOrNull(status?.durationSeconds), }; } function nativeArgoSummary(application: Record | null): Record | null { if (application === null) return null; const metadata = asOptionalRecord(application.metadata); const status = asOptionalRecord(application.status); const sync = asOptionalRecord(status?.sync); const health = asOptionalRecord(status?.health); return { name: stringOrNull(metadata?.name), namespace: stringOrNull(metadata?.namespace), syncStatus: stringOrNull(sync?.status), healthStatus: stringOrNull(health?.status), revision: stringOrNull(sync?.revision), ready: argoApplicationReady(application), }; } function nativeRuntimeSummary(runtime: NativeStatusSpec["runtime"], workloads: Record[], expectedSha: string | null): Record | null { if (runtime === null) return null; const targetSha = runtimeTargetShaFromWorkloads(runtime, workloads); return { namespace: runtime.namespace, ready: runtimeWorkloadsReady(runtime, workloads), targetSha, expectedSha, aligned: expectedSha !== null && targetSha !== null ? targetSha === expectedSha : null, workloads: runtime.workloads.map((spec, index) => { const workload = workloads[index] ?? null; const sourceCommit = workloadSourceCommit(spec, workload); return { kind: spec.kind, name: spec.name, ready: workloadReady(spec, workload), sourceCommit, aligned: expectedSha !== null && sourceCommit !== null ? sourceCommit === expectedSha : null, }; }), }; } function nativeGitMirrorGitopsBranch(follower: FollowerSpec): string | null { if (follower.adapter === "hwlab-node-runtime") { return hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node).gitopsBranch; } if (follower.adapter === "agentrun-yaml-lane") { return resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }).spec.gitops.branch; } return null; } function nativeGitMirrorRequired(follower: FollowerSpec): boolean { return nativeGitMirrorGitopsBranch(follower) !== null && (follower.closeoutChecks.includes("gitMirrorPostFlush") || follower.closeoutChecks.includes("gitops")); } function nativeGitMirrorReady(gitMirror: Record | null): boolean { if (gitMirror === null) return false; if (gitMirror.ok === false) return false; if (gitMirror.sourceSnapshotReady === false) return false; if (gitMirror.pendingFlush === true) return false; if (gitMirror.githubInSync === false) return false; return true; } function nativeGitMirrorSummary(gitMirror: Record | null): Record | null { if (gitMirror === null) return null; return { ok: gitMirror.ok === true, repository: stringOrNull(gitMirror.repository), sourceBranch: stringOrNull(gitMirror.sourceBranch), gitopsBranch: stringOrNull(gitMirror.gitopsBranch), localSource: stringOrNull(gitMirror.localSource), githubSource: stringOrNull(gitMirror.githubSource), sourceStageRef: stringOrNull(gitMirror.sourceStageRef), sourceSnapshotReady: gitMirror.sourceSnapshotReady === true, localGitops: stringOrNull(gitMirror.localGitops), githubGitops: stringOrNull(gitMirror.githubGitops), pendingFlush: gitMirror.pendingFlush === true, githubInSync: gitMirror.githubInSync === true, statusAuthority: stringOrNull(gitMirror.statusAuthority) ?? "k8s-git-mirror-cache", }; } function latestCondition(status: Record | null, type: string): Record | null { const conditions = Array.isArray(status?.conditions) ? status.conditions : []; for (const condition of conditions as unknown[]) { const record = asOptionalRecord(condition); if (record?.type === type) return record; } return null; } function mergeFollowerStatus( registry: BranchFollowerRegistry, follower: FollowerSpec, stored: Record, live: AdapterSummary | null, liveRequested: boolean, ): Record { const storedSource = asOptionalRecord(stored.source); const storedTarget = asOptionalRecord(stored.target); const phase = live?.phase ?? stringOrNull(stored.phase) ?? "Observed"; const observedSha = live?.observedSha ?? stringOrNull(storedSource?.observedSha); const targetSha = live?.targetSha ?? stringOrNull(storedTarget?.targetSha); const lastTriggeredSha = live?.lastTriggeredSha ?? stringOrNull(stored.lastTriggeredSha); const lastSucceededSha = live?.lastSucceededSha ?? stringOrNull(stored.lastSucceededSha); return { ok: live === null ? true : live.ok, id: follower.id, enabled: follower.enabled, adapter: follower.adapter, phase, source: { repository: follower.source.repository, branch: follower.source.branch, observedSha, snapshotPrefix: follower.source.snapshotPrefix, }, target: { node: follower.target.node, lane: follower.target.lane, namespace: follower.target.namespace, sentinel: follower.target.sentinel, targetSha, }, lastTriggeredSha, lastSucceededSha, pipelineRun: live?.pipelineRun ?? stringOrNull(stored.pipelineRun), inFlightJob: live?.inFlightJob ?? stringOrNull(stored.inFlightJob), budgetSource: follower.budgets, updatedAt: stringOrNull(stored.updatedAt), stateConfigMap: registry.controller.stateConfigMapName, live: liveRequested, message: live?.message ?? stringOrNull(stored.decision) ?? "no controller state yet", timings: live === null ? storedFollowerTimingsForStatus(follower, asOptionalRecord(stored.timings), phase, observedSha) : buildFollowerTimings(follower, live, undefined, asOptionalRecord(stored.timings), phase), warnings: Array.isArray(stored.warnings) ? stored.warnings.slice(0, 6) : [], next: followerNextCommands(follower), }; } function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead { const errors: string[] = []; const stateResult = kubeConfigMapFollowerState(registry, options); const namespace = registry.controller.namespace; const deploymentResult = kubeJson(registry, options, `kubectl -n ${shQuote(namespace)} get deploy ${shQuote(registry.controller.deploymentName)} -o json`, 10_000, `/apis/apps/v1/namespaces/${encodeURIComponent(namespace)}/deployments/${encodeURIComponent(registry.controller.deploymentName)}`); const leaseResult = kubeJson(registry, options, `kubectl -n ${shQuote(namespace)} get lease ${shQuote(registry.controller.leaseName)} -o json`, 10_000, `/apis/coordination.k8s.io/v1/namespaces/${encodeURIComponent(namespace)}/leases/${encodeURIComponent(registry.controller.leaseName)}`); const podSelector = labelSelector(registry.controller.labels); const podsResult = kubePodList(registry, options, podSelector); if (!stateResult.ok) errors.push(`state configmap: ${stateResult.error}`); if (!deploymentResult.ok && !isNotFoundText(deploymentResult.error)) errors.push(`deployment: ${deploymentResult.error}`); if (!leaseResult.ok && !isNotFoundText(leaseResult.error)) errors.push(`lease: ${leaseResult.error}`); if (!podsResult.ok && !isNotFoundText(podsResult.error)) errors.push(`pods: ${podsResult.error}`); return { ok: errors.length === 0, stateByFollower: stateResult.stateByFollower, stateMetadata: stateResult.metadata, stateValueBytes: stateResult.valueBytes, stateConfigMapPresent: stateResult.present, deployment: deploymentResult.value, lease: leaseResult.value, pods: podsResult.value, errors, }; } function kubeConfigMapFollowerState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sFollowerStateRead { const script = [ "set -eu", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", nativeCicdScriptLoadShell(["read-state-summary.mjs"]), `NAMESPACE=${shQuote(registry.controller.namespace)}`, `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, `FOLLOWERS_JSON=${shQuote(JSON.stringify(registry.followers.map((follower) => follower.id)))}`, "MAX_TIMING_STAGES=24", "export NAMESPACE CONFIGMAP FOLLOWERS_JSON MAX_TIMING_STAGES", "node \"$tmpdir/read-state-summary.mjs\"", ].join("\n"); const result = runKubeScript(registry, options, script, "", 10_000); const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; if (parsed === null) { const error = redactText(tailText(result.stderr || result.stdout, 800)); return { ok: false, stateByFollower: {}, present: false, error }; } const parsedStates = asOptionalRecord(parsed.stateByFollower) ?? {}; const metadata = asOptionalRecord(parsed.metadata); const parsedValueBytes = asOptionalRecord(parsed.valueBytes) ?? {}; const stateByFollower: Record> = {}; const valueBytes: Record = {}; for (const follower of registry.followers) { const state = asOptionalRecord(parsedStates[follower.id]); if (state !== null) stateByFollower[follower.id] = state; const bytes = numberOrNull(parsedValueBytes[follower.id]); if (bytes !== null) valueBytes[follower.id] = bytes; } const errors = Array.isArray(parsed.errors) ? parsed.errors.map(String).filter((item) => item.length > 0) : []; return { ok: parsed.ok === true && errors.length === 0, stateByFollower, metadata, valueBytes, present: parsed.present === true, error: errors.join("; "), }; } function stateWriteSummary(followerId: string, result: CommandResult): Record { const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; return { follower: followerId, ok: result.exitCode === 0 && parsed?.ok !== false, exitCode: result.exitCode, timedOut: result.timedOut, beforeResourceVersion: stringOrNull(parsed?.beforeResourceVersion), afterResourceVersion: stringOrNull(parsed?.afterResourceVersion), preservedTiming: parsed?.preservedTiming === true, message: result.exitCode === 0 ? "state patch command completed" : redactText(tailText(result.stderr || result.stdout, 500)), parsedDownstreamCliOutput: false, }; } function removeFollowerStateKeys(registry: BranchFollowerRegistry, options: ParsedOptions, ids: string[]): CommandResult { const patch = JSON.stringify({ data: Object.fromEntries(ids.map((id) => [id, null])) }); const script = options.inCluster ? [ "set -eu", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", nativeCicdScriptLoadShell(["patch-configmap-data.mjs"]), `NAMESPACE=${shQuote(registry.controller.namespace)}`, `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, `PATCH_B64=${shQuote(Buffer.from(patch, "utf8").toString("base64"))}`, "export NAMESPACE CONFIGMAP PATCH_B64", "node \"$tmpdir/patch-configmap-data.mjs\"", ].join("\n") : [ "set -eu", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", `NAMESPACE=${shQuote(registry.controller.namespace)}`, `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, `PATCH=${shQuote(patch)}`, "export NAMESPACE CONFIGMAP PATCH", "if ! kubectl -n \"$NAMESPACE\" get configmap \"$CONFIGMAP\" >/dev/null 2>\"$tmpdir/error\"; then", " if grep -qi 'not found' \"$tmpdir/error\"; then", " printf '{\"ok\":true,\"present\":false,\"patched\":false,\"reason\":\"state-configmap-not-found\",\"parsedDownstreamCliOutput\":false}'", " exit 0", " fi", " cat \"$tmpdir/error\" >&2", " exit 1", "fi", "kubectl -n \"$NAMESPACE\" patch configmap \"$CONFIGMAP\" --type merge -p \"$PATCH\" >/dev/null", "printf '{\"ok\":true,\"present\":true,\"patched\":true,\"parsedDownstreamCliOutput\":false}'", ].join("\n"); return runKubeScript(registry, options, script, "", 10_000); } function kubeJson(registry: BranchFollowerRegistry, options: ParsedOptions, command: string, timeoutMs: number, inClusterPath?: string): { ok: boolean; value: Record | null; error: string } { const script = options.inCluster && inClusterPath !== undefined ? [ "set -eu", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", nativeCicdScriptLoadShell(["kube-get.mjs"]), `node "$tmpdir/kube-get.mjs" ${shQuote(inClusterPath)}`, ].join("\n") : `set -eu\n${command}`; const result = runKubeScript(registry, options, script, "", timeoutMs); const value = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; return { ok: result.exitCode === 0 && value !== null, value, error: redactText(tailText(result.stderr || result.stdout, 800)), }; } function kubePodList(registry: BranchFollowerRegistry, options: ParsedOptions, selector: string): { ok: boolean; value: Record | null; error: string } { const command = `kubectl -n ${shQuote(registry.controller.namespace)} get pods -l ${shQuote(selector)} -o name`; const script = options.inCluster ? [ "set -eu", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", nativeCicdScriptLoadShell(["kube-get.mjs"]), `node "$tmpdir/kube-get.mjs" ${shQuote(`/api/v1/namespaces/${encodeURIComponent(registry.controller.namespace)}/pods?labelSelector=${encodeURIComponent(selector)}`)}`, ].join("\n") : `set -eu\n${command}`; const result = runKubeScript(registry, options, script, "", 10_000); const parsed = options.inCluster && result.exitCode === 0 ? parseJsonObject(result.stdout) : null; const names = options.inCluster ? arrayRecords(parsed?.items).map((item) => stringOrNull(asOptionalRecord(item.metadata)?.name)).filter((name): name is string => name !== null) : result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => line.replace(/^pod\//u, "")); return { ok: result.exitCode === 0, value: result.exitCode === 0 ? { items: names.map((name) => ({ metadata: { name } })) } : null, error: redactText(tailText(result.stderr || result.stdout, 800)), }; } function runKubeScript(registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number): CommandResult { if (options.inCluster) { return runCommand(["sh", "-lc", script], repoRoot, { input, timeoutMs }); } return runCommand([transPath(), registry.controller.kubeRoute, "sh"], repoRoot, { input: `${script}\n`, timeoutMs }); } function compactFollowerStateForConfigMap(state: FollowerState): Record { return { id: state.id, adapter: state.adapter, enabled: state.enabled, phase: state.phase, source: state.source, target: state.target, lastTriggeredSha: state.lastTriggeredSha, lastSucceededSha: state.lastSucceededSha, pipelineRun: state.pipelineRun, inFlightJob: state.inFlightJob, budgetSource: state.budgetSource, controller: state.controller, decision: state.decision, dryRun: state.dryRun, updatedAt: state.updatedAt, timings: compactTimings(state.timings), warnings: compactStateWarnings(state.warnings), stateFormat: "compact-v1", command: compactStateCommand(state.command), }; } function compactStateCommand(command: Record | undefined): Record | undefined { if (command === undefined) return undefined; const closeout = asOptionalRecord(command.closeout); const closeoutSummary = asOptionalRecord(closeout?.summary); const payload = compactNativePayload(asOptionalRecord(command.payload)) ?? closeoutSummary; return { mode: stringOrNull(command.mode) ?? stringOrNull(command.status), namespace: stringOrNull(command.namespace), pipelineRun: stringOrNull(command.pipelineRun), sourceCommit: stringOrNull(command.sourceCommit), sourceStageRef: stringOrNull(command.sourceStageRef), wait: command.wait === true ? true : undefined, pipelineRunCompleted: command.pipelineRunCompleted === true ? true : undefined, stillRunning: command.stillRunning === true ? true : undefined, closeout: closeout === null ? null : { ok: closeout.ok === true, completed: closeout.completed === true, timedOut: closeout.timedOut === true, polls: numberOrNull(closeout.polls), elapsedMs: numberOrNull(closeout.elapsedMs), gitMirrorFlush: compactCloseoutGitMirrorFlush(asOptionalRecord(closeout.gitMirrorFlush)), summary: closeoutSummary, statusAuthority: stringOrNull(closeout.statusAuthority), parsedDownstreamCliOutput: false, }, payload, exitCode: numberOrNull(command.exitCode), timedOut: command.timedOut === true, statusAuthority: stringOrNull(command.statusAuthority), parsedDownstreamCliOutput: false, }; } function compactCloseoutGitMirrorFlush(value: Record | null): Record | null { if (value === null) return null; const result = asOptionalRecord(value.result); return { jobName: stringOrNull(value.jobName), namespace: stringOrNull(value.namespace), result: result === null ? null : { ok: result.ok === true, completed: result.completed === true, failed: result.failed === true, timedOut: result.timedOut === true, created: result.created === true, reused: result.reused === true, jobName: stringOrNull(result.jobName), namespace: stringOrNull(result.namespace), elapsedMs: numberOrNull(result.elapsedMs), conditionReason: stringOrNull(result.conditionReason), conditionMessage: stringOrNull(result.conditionMessage), statusAuthority: stringOrNull(result.statusAuthority), parsedDownstreamCliOutput: false, }, }; } function compactStateWarnings(warnings: string[]): string[] { return warnings.slice(0, 4).map((item) => redactText(item.length <= 800 ? item : `${item.slice(0, 400)} ... ${item.slice(-400)}`)); } function compactNativePayload(payload: Record | null): Record | null { if (payload === null) return null; return { source: compactSourcePayload(asOptionalRecord(payload.source)), sourceSync: compactSourceSyncPayload(asOptionalRecord(payload.sourceSync)), gitMirror: asOptionalRecord(payload.gitMirror), tekton: asOptionalRecord(payload.tekton), taskRuns: compactTaskRunsPayload(asOptionalRecord(payload.taskRuns)), planArtifacts: compactPlanArtifactsPayload(asOptionalRecord(payload.planArtifacts)), argo: asOptionalRecord(payload.argo), runtime: asOptionalRecord(payload.runtime), errors: Array.isArray(payload.errors) ? payload.errors.slice(0, 5) : [], statusAuthority: stringOrNull(payload.statusAuthority), parsedDownstreamCliOutput: false, }; } function compactSourceSyncPayload(value: Record | null): Record | null { if (value === null) return null; return { ok: value.ok === true, completed: value.completed === true, failed: value.failed === true, timedOut: value.timedOut === true, created: value.created === true, reused: value.reused === true, jobName: stringOrNull(value.jobName), namespace: stringOrNull(value.namespace), elapsedMs: numberOrNull(value.elapsedMs), conditionReason: stringOrNull(value.conditionReason), conditionMessage: stringOrNull(value.conditionMessage), statusAuthority: stringOrNull(value.statusAuthority), parsedDownstreamCliOutput: false, }; } function compactTaskRunsPayload(taskRuns: Record | null): Record | null { if (taskRuns === null) return null; return { ok: taskRuns.ok === true, count: numberOrNull(taskRuns.count), succeededCount: numberOrNull(taskRuns.succeededCount), failedCount: numberOrNull(taskRuns.failedCount), activeCount: numberOrNull(taskRuns.activeCount), performance: asOptionalRecord(taskRuns.performance), items: Array.isArray(taskRuns.items) ? taskRuns.items.slice(0, 16) : [], }; } function compactPlanArtifactsPayload(planArtifacts: Record | null): Record | null { if (planArtifacts === null) return null; return { ok: planArtifacts.ok === true, pipelineRun: stringOrNull(planArtifacts.pipelineRun), eventFound: planArtifacts.eventFound === true, degradedReason: stringOrNull(planArtifacts.degradedReason), sourceCommitId: stringOrNull(planArtifacts.sourceCommitId), affectedServices: Array.isArray(planArtifacts.affectedServices) ? planArtifacts.affectedServices.slice(0, 40) : [], rolloutServices: Array.isArray(planArtifacts.rolloutServices) ? planArtifacts.rolloutServices.slice(0, 40) : [], buildServices: Array.isArray(planArtifacts.buildServices) ? planArtifacts.buildServices.slice(0, 40) : [], reusedServices: Array.isArray(planArtifacts.reusedServices) ? planArtifacts.reusedServices.slice(0, 40) : [], buildSkippedCount: numberOrNull(planArtifacts.buildSkippedCount), summary: stringOrNull(planArtifacts.summary), disclosure: stringOrNull(planArtifacts.disclosure), }; } function compactSourcePayload(source: Record | null): Record | null { if (source === null) return null; return { commit: stringOrNull(source.commit), branch: stringOrNull(source.branch), repository: stringOrNull(source.repository), stageRef: stringOrNull(source.stageRef), sourceAuthority: stringOrNull(source.sourceAuthority), repoPath: stringOrNull(source.repoPath), }; } function buildFollowerTimings( follower: FollowerSpec, live: AdapterSummary, triggerCommand: Record | undefined, storedTimings?: Record | null, phase?: BranchFollowerPhase, ): FollowerState["timings"] { const nativePayload = asOptionalRecord(live.payload); const finishOverride = stringOrNull(triggerCommand?.finishedAt) ?? noopStoredTotalFinishOverride(storedTimings, phase, live); const total = totalTimingFromCommand(triggerCommand, phase) ?? totalTimingFromStored(storedTimings, phase, finishOverride, live.observedSha); const stages = dedupeTimingStages([ ...stageTimingsFromCommand(triggerCommand), ...stageTimingsFromNativePayload(nativePayload), ]).slice(0, 24); const stageSourceCommit = stages.length > 0 ? live.observedSha : null; return { budgetSeconds: follower.budgets.endToEndSeconds, totalSeconds: total?.seconds ?? null, totalStatus: total?.status ?? "unknown", totalSource: total?.source ?? "-", sourceCommit: total?.sourceCommit ?? stringOrNull(triggerCommand?.sourceCommit) ?? stageSourceCommit, startedAt: total?.startedAt ?? null, finishedAt: total?.finishedAt ?? null, overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, stages, }; } function noopStoredTotalFinishOverride( storedTimings: Record | null | undefined, phase: BranchFollowerPhase | undefined, live: AdapterSummary, ): string | null { if (phase !== "Noop" || live.aligned !== true || live.observedSha === null) return null; if (storedTimings === null || storedTimings === undefined) return null; if (stringOrNull(storedTimings.sourceCommit) !== live.observedSha) return null; if (stringOrNull(storedTimings.startedAt) === null) return null; if (stringOrNull(storedTimings.finishedAt) !== null) return null; return new Date().toISOString(); } function storedFollowerTimingsForStatus( follower: FollowerSpec, storedTimings: Record | null, phase: BranchFollowerPhase, observedSha: string | null, ): FollowerState["timings"] { const total = totalTimingFromStored(storedTimings, phase, null, observedSha); const sourceCommit = total?.sourceCommit ?? stringOrNull(storedTimings?.sourceCommit) ?? null; return { budgetSeconds: follower.budgets.endToEndSeconds, totalSeconds: total?.seconds ?? null, totalStatus: total?.status ?? "unknown", totalSource: total?.source ?? "-", sourceCommit, startedAt: total?.startedAt ?? null, finishedAt: total?.finishedAt ?? null, overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, stages: sourceCommit === null ? [] : storedStageTimings(storedTimings), }; } function storedStageTimings(storedTimings: Record | null): StageTiming[] { if (storedTimings === null) return []; return arrayRecords(storedTimings.stages) .map((stage) => stageTiming( stringOrNull(stage.stage) ?? "", stringOrNull(stage.status) ?? "unknown", numberOrNull(stage.seconds), numberOrNull(stage.budgetSeconds), stringOrNull(stage.source) ?? "stored-state", stringOrNull(stage.object), )) .filter((stage) => stage.stage.length > 0) .slice(0, 24); } function compactTimings(timings: FollowerState["timings"]): FollowerState["timings"] { return { budgetSeconds: timings.budgetSeconds, totalSeconds: timings.totalSeconds, totalStatus: timings.totalStatus, totalSource: timings.totalSource, sourceCommit: timings.sourceCommit, startedAt: timings.startedAt, finishedAt: timings.finishedAt, overBudget: timings.overBudget, stages: timings.stages.slice(0, 24).map((stage) => ({ stage: stage.stage, status: stage.status, seconds: stage.seconds, budgetSeconds: stage.budgetSeconds, source: stage.source, object: stage.object, })), }; } function totalTimingFromCommand(command: Record | undefined, phase?: BranchFollowerPhase): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null { if (command === undefined) return null; if (command.mode === "k8s-native-closeout") return null; const payload = asOptionalRecord(command.payload); if (payload?.reused === true) return null; const startedAt = stringOrNull(command.startedAt); const finishedAt = stringOrNull(command.finishedAt); const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? secondsFromMs(numberOrNull(command.elapsedMs)); if (seconds === null) return null; const closeout = asOptionalRecord(command.closeout); const exitCode = numberOrNull(command.exitCode); const status = command.ok === false || (exitCode !== null && exitCode !== 0) ? "failed" : command.timedOut === true || closeout?.timedOut === true ? "timed-out" : closeout?.completed === true || command.completed === true ? "completed" : command.stillRunning === true ? "running" : command.pipelineRunCompleted === true ? "ci-completed" : phase === undefined ? "submitted" : phase.toLowerCase(); return { seconds, status, source: stringOrNull(command.mode) ?? stringOrNull(command.status) ?? "command", sourceCommit: stringOrNull(command.sourceCommit), startedAt, finishedAt }; } function totalTimingFromStored(storedTimings: Record | null | undefined, phase?: BranchFollowerPhase, finishOverride?: string | null, observedSha?: string | null): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null { if (storedTimings === null || storedTimings === undefined) return null; const status = stringOrNull(storedTimings.totalStatus); const source = stringOrNull(storedTimings.totalSource); const sourceCommit = stringOrNull(storedTimings.sourceCommit); if (sourceCommit === null) return null; if (observedSha !== null && observedSha !== undefined && sourceCommit !== observedSha) return null; const startedAt = stringOrNull(storedTimings.startedAt); const finishedAt = stringOrNull(storedTimings.finishedAt) ?? finishOverride ?? null; if (phase === "Noop" && finishedAt === null) return null; const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? numberOrNull(storedTimings.totalSeconds); if (seconds === null) return null; return { seconds, status: finishedAt === null && phase !== undefined && !terminalPhase(phase) ? phase.toLowerCase() : phase === undefined ? status ?? "recorded" : phase.toLowerCase(), source: source ?? "stored-state", sourceCommit, startedAt, finishedAt, }; } function totalSecondsFromRange(startedAt: string | null, finishedAt: string | null): number | null { const startedMs = timestampMs(startedAt); if (startedMs === null) return null; const finishedMs = timestampMs(finishedAt) ?? Date.now(); return finishedMs >= startedMs ? roundSeconds((finishedMs - startedMs) / 1000) : null; } function timestampMs(value: string | null): number | null { if (value === null) return null; const parsed = Date.parse(value); return Number.isFinite(parsed) ? parsed : null; } function terminalPhase(phase: BranchFollowerPhase): boolean { return phase === "Succeeded" || phase === "Failed" || phase === "Blocked" || phase === "Skipped" || phase === "Noop"; } function stageTimingsFromNativePayload(payload: Record | null): StageTiming[] { if (payload === null) return []; const stages: StageTiming[] = []; const statusRead = asOptionalRecord(asOptionalRecord(payload.timings)?.statusRead); stages.push(stageTiming("status-read", "ok", secondsFromMs(numberOrNull(statusRead?.elapsedMs)), numberOrNull(statusRead?.budgetSeconds), "native-status", null)); const sourceSyncStage = k8sJobTiming("git-mirror-sync", asOptionalRecord(payload.sourceSync)); if (sourceSyncStage !== null) stages.push(sourceSyncStage); const gitMirror = asOptionalRecord(payload.gitMirror); if (gitMirror !== null) { const status = gitMirror.pendingFlush === true ? "pending-flush" : gitMirror.githubInSync === true && gitMirror.sourceSnapshotReady === true ? "ready" : "not-ready"; stages.push(stageTiming("git-mirror", status, null, null, "git-mirror-cache", stringOrNull(gitMirror.gitopsBranch) ?? stringOrNull(gitMirror.sourceBranch))); } const tekton = asOptionalRecord(payload.tekton); if (tekton !== null) { const status = tekton.succeeded === true ? "succeeded" : tekton.succeeded === false ? `failed:${stringOrNull(tekton.reason) ?? "unknown"}` : "running"; stages.push(stageTiming("pipelinerun", status, numberOrNull(tekton.durationSeconds), null, "tekton", stringOrNull(tekton.name))); } const taskRuns = asOptionalRecord(payload.taskRuns); const taskRunItems = taskRuns !== null && Array.isArray(taskRuns.items) ? taskRuns.items : []; for (const item of taskRunItems) { const record = asOptionalRecord(item); if (record === null) continue; const name = stringOrNull(record.pipelineTask) ?? stringOrNull(record.name) ?? "unknown"; const status = record.status === "True" ? "succeeded" : record.status === "False" ? `failed:${stringOrNull(record.reason) ?? "unknown"}` : "running"; stages.push(stageTiming(`task:${name}`, status, numberOrNull(record.durationSeconds), null, "tekton-taskrun", stringOrNull(record.name))); } const argo = asOptionalRecord(payload.argo); if (argo !== null) { stages.push(stageTiming("argo", `${stringOrNull(argo.syncStatus) ?? "unknown"}/${stringOrNull(argo.healthStatus) ?? "unknown"}`, null, null, "argocd", stringOrNull(argo.name))); } const runtime = asOptionalRecord(payload.runtime); if (runtime !== null) { const aligned = runtime.aligned === true ? "aligned" : runtime.aligned === false ? "stale" : "unknown-target"; stages.push(stageTiming("runtime", `${runtime.ready === true ? "ready" : "not-ready"}/${aligned}`, null, null, "kubernetes-workload", stringOrNull(runtime.namespace))); } return stages; } function stageTimingsFromCommand(command: Record | undefined): StageTiming[] { if (command === undefined) return []; const stages: StageTiming[] = []; const phase = stringOrNull(command.phase); const jobStage = phase === null ? null : k8sJobTiming(phase, asOptionalRecord(command.job), stringOrNull(command.jobName)); if (jobStage !== null) stages.push(jobStage); const payload = asOptionalRecord(command.payload); if (payload !== null) { const capabilities = asOptionalRecord(payload.nativeCapabilities); for (const stage of [ k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)), k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)), k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)), ]) { if (stage !== null) stages.push(stage); } const agentrun = asOptionalRecord(payload.agentrun); const agentrunSync = asOptionalRecord(agentrun?.gitMirrorSync); const agentrunFlush = asOptionalRecord(agentrun?.gitMirrorFlush); const imageBuild = asOptionalRecord(agentrun?.imageBuild); const gitopsPublish = asOptionalRecord(agentrun?.gitopsPublish); for (const stage of [ k8sJobTiming("git-mirror-sync", asOptionalRecord(agentrunSync?.payload), stringOrNull(agentrunSync?.jobName)), k8sJobTiming("image-build", asOptionalRecord(imageBuild?.result), stringOrNull(imageBuild?.jobName)), k8sJobTiming("gitops-publish", asOptionalRecord(gitopsPublish?.result), stringOrNull(gitopsPublish?.jobName)), k8sJobTiming("git-mirror-flush", asOptionalRecord(agentrunFlush?.payload), stringOrNull(agentrunFlush?.jobName)), ]) { if (stage !== null) stages.push(stage); } const tektonSeconds = secondsFromMs(numberOrNull(payload.elapsedMs)); if (tektonSeconds !== null) { const status = payload.completed === true ? "completed" : payload.failed === true ? "failed" : payload.stillRunning === true ? "running" : "submitted"; stages.push(stageTiming("pipelinerun-wait", status, tektonSeconds, null, "tekton-submit", stringOrNull(command.pipelineRun))); } } const closeout = asOptionalRecord(command.closeout); if (closeout !== null) { const gitMirrorFlush = asOptionalRecord(closeout.gitMirrorFlush); const gitMirrorFlushStage = k8sJobTiming("git-mirror-flush", asOptionalRecord(gitMirrorFlush?.result), stringOrNull(gitMirrorFlush?.jobName)); if (gitMirrorFlushStage !== null) stages.push(gitMirrorFlushStage); const status = closeout.completed === true ? "completed" : closeout.timedOut === true ? "timed-out" : "pending"; stages.push(stageTiming("closeout", status, secondsFromMs(numberOrNull(closeout.elapsedMs)), null, "k8s-native-closeout", stringOrNull(command.pipelineRun))); } return stages; } function k8sJobTiming(stage: string, job: Record | null, objectOverride?: string | null): StageTiming | null { if (job === null) return null; const status = job.completed === true ? job.reused === true ? "reused" : "completed" : job.failed === true ? "failed" : job.timedOut === true ? "timed-out" : "running"; return stageTiming(stage, status, secondsFromMs(numberOrNull(job.elapsedMs)), null, "kubernetes-job", objectOverride ?? stringOrNull(job.jobName)); } function stageTiming(stage: string, status: string, seconds: number | null, budgetSeconds: number | null, source: string, object: string | null): StageTiming { return { stage, status, seconds, budgetSeconds, source, object }; } function dedupeTimingStages(stages: StageTiming[]): StageTiming[] { const byKey = new Map(); for (const stage of stages) { if (stage.stage.length === 0) continue; const key = `${stage.stage}\t${stage.object ?? ""}`; const previous = byKey.get(key); if (previous === undefined || previous.seconds === null && stage.seconds !== null) byKey.set(key, stage); } return [...byKey.values()]; } function secondsFromMs(value: number | null): number | null { return value === null ? null : roundSeconds(value / 1000); } function roundSeconds(value: number): number { return Math.round(value * 10) / 10; } function writeFollowerState(registry: BranchFollowerRegistry, state: FollowerState, options: ParsedOptions): CommandResult { const stateJson = JSON.stringify(compactFollowerStateForConfigMap(state)); const script = [ "set -eu", `NAMESPACE=${shQuote(registry.controller.namespace)}`, `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, `FOLLOWER_ID=${shQuote(state.id)}`, `SPEC_REF=${shQuote(SPEC_REF)}`, `STATE_B64=${shQuote(Buffer.from(stateJson, "utf8").toString("base64"))}`, "export NAMESPACE CONFIGMAP FOLLOWER_ID SPEC_REF STATE_B64", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", nativeCicdScriptLoadShell(["patch-follower-state.mjs"]), "node \"$tmpdir/patch-follower-state.mjs\"", ].join("\n"); return runKubeScript(registry, options, script, "", 10_000); } function runControllerReconcileJob(registry: BranchFollowerRegistry, options: ParsedOptions, mode: { dryRun: boolean; wait: boolean; recordState: boolean }): Record { const timeoutSeconds = options.timeoutSeconds ?? registry.controller.budgets.runOnceSeconds; const jobName = `${registry.controller.deploymentName}-once-${Date.now().toString(36)}`.slice(0, 63); const manifest = renderControllerReconcileJob(registry, options, jobName, mode, timeoutSeconds); const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`; const manifestBase64 = Buffer.from(manifestYaml, "utf8").toString("base64"); const script = [ "set -eu", "tmp=$(mktemp)", "base64 -d >\"$tmp\" <<'UNIDESK_CICD_RECONCILE_JOB_B64'", manifestBase64, "UNIDESK_CICD_RECONCILE_JOB_B64", `kubectl -n ${shQuote(registry.controller.namespace)} delete job ${shQuote(jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`, `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`, mode.wait ? waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds) : "true", ].join("\n"); const result = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000); return { ok: result.exitCode === 0, name: jobName, namespace: registry.controller.namespace, mode: mode.dryRun ? "status-refresh" : "confirm-reconcile", execution: "k8s-native-job", exitCode: result.exitCode, timedOut: result.timedOut, message: result.exitCode === 0 ? "reconcile job completed" : redactText(tailText(result.stderr || result.stdout, 800)), stdoutTail: redactText(tailText(result.stdout, options.full ? 4000 : 1000)), stderrTail: redactText(tailText(result.stderr, options.full ? 2000 : 800)), }; } function selectFollowers(registry: BranchFollowerRegistry, options: ParsedOptions, opts: { includeDisabled: boolean }): FollowerSpec[] { let selected = registry.followers; if (options.followerId !== null) selected = selected.filter((item) => item.id === options.followerId); else if (!options.all && options.action === "run-once") selected = selected.filter((item) => item.enabled); if (!opts.includeDisabled) selected = selected.filter((item) => item.enabled); if (selected.length === 0) throw new Error(options.followerId === null ? "no followers selected" : `unknown or disabled follower ${options.followerId}`); return selected; } function registrySummary(registry: BranchFollowerRegistry): Record { return { path: registry.path, sha256: registry.rawSha256, metadata: registry.metadata, controller: { namespace: registry.controller.namespace, kubeRoute: registry.controller.kubeRoute, deploymentName: registry.controller.deploymentName, stateConfigMapName: registry.controller.stateConfigMapName, leaseName: registry.controller.leaseName, }, followers: registry.followers.map((item) => item.id), }; } function redactCommands(follower: FollowerSpec): Record { return { plan: follower.commands.plan.argv.join(" "), status: "native:k8s-git-mirror+tekton+argocd+runtime", trigger: follower.commands.trigger.argv.join(" "), events: follower.commands.events.argv.join(" "), logs: follower.commands.logs.argv.join(" "), }; } function nativeStatusPlan(native: NativeStatusSpec): Record { return { source: { mode: "k8s-git-mirror-cache", gitMirrorNamespace: native.source.gitMirrorNamespace, gitMirrorDeployment: native.source.gitMirrorDeployment, repoPath: native.source.repoPath, }, tekton: native.tekton, argo: native.argo, runtime: native.runtime === null ? null : { namespace: native.runtime.namespace, workloads: native.runtime.workloads.map((item) => ({ kind: item.kind, name: item.name })), }, parsedDownstreamCliOutput: false, }; } function controllerStatusSummary(registry: BranchFollowerRegistry, k8s: K8sStateRead): Record { const deploymentStatus = asOptionalRecord(k8s.deployment?.status); const available = numberOrNull(deploymentStatus?.availableReplicas) ?? 0; const replicas = numberOrNull(deploymentStatus?.replicas) ?? 0; const leaseSpec = asOptionalRecord(k8s.lease?.spec); const podItems = Array.isArray(k8s.pods?.items) ? k8s.pods.items.length : null; return { namespace: registry.controller.namespace, route: registry.controller.kubeRoute, deploymentName: registry.controller.deploymentName, deploymentPresent: k8s.deployment !== null, availableReplicas: available, replicas, pods: podItems, stateConfigMapName: registry.controller.stateConfigMapName, stateConfigMapPresent: k8s.stateConfigMapPresent, leaseName: registry.controller.leaseName, leaseHolder: stringOrNull(leaseSpec?.holderIdentity), noHostWorktreeAuthority: true, }; } function followerNextCommands(follower: FollowerSpec): Record { const next: Record = { status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, liveStatus: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --live`, dryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`, trigger: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --confirm --wait`, events: `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`, logs: `bun scripts/cli.ts cicd branch-follower logs --follower ${follower.id}`, }; if (follower.nativeStatus.tekton !== null) next.pipelineRuns = `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`; if (follower.nativeStatus.argo !== null) next.argoApplication = `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`; return next; } function safeResolveString(ref: string): string | null { try { return resolveConfigRefString(ref, ref); } catch { return null; } } function parseJsonObject(text: string): Record | null { const trimmed = text.trim(); if (trimmed.length === 0) return null; try { const parsed = JSON.parse(trimmed) as unknown; return asOptionalRecord(parsed); } catch { const start = trimmed.indexOf("{"); const end = trimmed.lastIndexOf("}"); if (start < 0 || end <= start) return null; try { const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown; return asOptionalRecord(parsed); } catch { return null; } } } function recordAt(root: Record, path: string[]): Record | null { let current: unknown = root; for (const item of path) { if (typeof current !== "object" || current === null || Array.isArray(current)) return null; current = (current as Record)[item]; } return asOptionalRecord(current); } function asOptionalRecord(value: unknown): Record | null { if (typeof value !== "object" || value === null || Array.isArray(value)) return null; return value as Record; } function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } function numberOrNull(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } function commandCompact(result: CommandResult, options: ParsedOptions): Record { return { argv: result.command, exitCode: result.exitCode, timedOut: result.timedOut, stdoutBytes: Buffer.byteLength(result.stdout), stderrBytes: Buffer.byteLength(result.stderr), stdoutTail: options.full || result.exitCode !== 0 ? redactText(tailText(result.stdout, options.tailBytes)) : "", stderrTail: options.full || result.exitCode !== 0 ? redactText(tailText(result.stderr, Math.min(options.tailBytes, 4000))) : "", }; } function objectRef(item: Record): Record { const metadata = asRecord(item.metadata, "metadata"); return { kind: stringField(item, "kind", "manifest"), namespace: typeof metadata.namespace === "string" ? metadata.namespace : "-", name: stringField(metadata, "name", "manifest.metadata"), }; } function labelSelector(labels: Record): string { return Object.entries(labels).map(([key, value]) => `${key}=${value}`).join(","); } function isNotFoundText(value: string): boolean { return /notfound|not found|notfound|NotFound/u.test(value); } function shortSha(value: string | null): string { if (value === null) return "-"; return value.length > 12 ? value.slice(0, 12) : value; } function safeJobSegment(value: string): string { return value.replace(/[^A-Za-z0-9_.-]/gu, "_").slice(0, 60); } function nativeCapabilityJobName(followerId: string, action: string, sha: string): string { const prefix = `${safeK8sNameSegment(followerId)}-${safeK8sNameSegment(action)}`; return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63); } function safeK8sNameSegment(value: string): string { const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, ""); return (normalized.length === 0 ? "x" : normalized).slice(0, 40).replace(/-$/u, ""); } function tailText(text: string, maxChars: number): string { if (text.length <= maxChars) return text; return text.slice(text.length - maxChars); } function commandLabel(options: ParsedOptions): string { return `cicd branch-follower ${options.action}`; } function renderResult(command: string, payload: Record, options: ParsedOptions): RenderedCliResult { const ok = payload.ok !== false; if (options.output === "json") return renderMachine(command, payload, "json", ok); if (options.output === "yaml") return renderMachine(command, payload, "yaml", ok); return rendered(ok, command, renderHuman(command, payload, options)); } function renderMachine(command: string, value: unknown, mode: "json" | "yaml", ok = true): RenderedCliResult { return rendered(ok, command, mode === "json" ? `${JSON.stringify(value, null, 2)}\n` : `${Bun.YAML.stringify(value)}\n`, mode === "json" ? "application/json" : "application/yaml"); } function rendered(ok: boolean, command: string, renderedText: string, contentType: RenderedCliResult["contentType"] = "text/plain"): RenderedCliResult { return { ok, command, renderedText, contentType }; } function renderHuman(command: string, payload: Record, options: ParsedOptions): string { if (command.endsWith(" plan")) return renderPlanHuman(payload); if (command.endsWith(" apply")) return renderApplyHuman(payload); if (command.endsWith(" status")) return renderStatusHuman(payload, options); if (command.endsWith(" run-once")) return renderRunOnceHuman(payload); if (command.endsWith(" debug-step")) return renderDebugStepHuman(payload); if (command.endsWith(" cleanup-state")) return renderCleanupStateHuman(payload); if (command.endsWith(" events") || command.endsWith(" logs")) return renderDrillDownHuman(payload); return `${JSON.stringify(payload, null, 2)}\n`; } function renderPlanHuman(payload: Record): string { const followers = arrayRecords(payload.followers); const rows = followers.map((item) => { const source = asOptionalRecord(item.source); const target = asOptionalRecord(item.target); const budgets = asOptionalRecord(item.budgets); return [ item.id, item.enabled, item.adapter, `${source?.repository ?? "-"}@${source?.branch ?? "-"}`, `${target?.node ?? "-"}/${target?.lane ?? "-"}`, budgets?.endToEndSeconds ?? "-", arrayRecords(item.configRefGraph).length, arrayText(item.closeoutChecks), ]; }); const next = asOptionalRecord(payload.next); return [ `CI/CD BRANCH-FOLLOWER PLAN (${payload.ok === false ? "blocked" : "ok"})`, "", table(["FOLLOWER", "ENABLED", "ADAPTER", "SOURCE", "TARGET", "BUDGET", "REFS", "CHECKS"], rows), "", "SOURCE AUTHORITY", `hostWorktreeAuthority=${payload.hostWorktreeAuthority === true ? "true" : "false"} mode=${asOptionalRecord(payload.sourceAuthority)?.mode ?? "-"} resolver=${asOptionalRecord(payload.sourceAuthority)?.resolver ?? "-"}`, "", "NEXT", `apply: ${next?.apply ?? "-"}`, `status: ${next?.status ?? "-"}`, `dry-run: ${next?.dryRun ?? "-"}`, "", ].join("\n"); } function renderApplyHuman(payload: Record): string { const controller = asOptionalRecord(payload.controller); const command = asOptionalRecord(payload.command); const next = asOptionalRecord(payload.next); return [ `CI/CD BRANCH-FOLLOWER APPLY (${payload.ok === false ? "failed" : payload.dryRun === true ? "dry-run" : "ok"})`, "", table( ["NAMESPACE", "ROUTE", "DEPLOYMENT", "STATE_CM", "LEASE", "HOST_WORKTREE"], [[controller?.namespace ?? "-", controller?.route ?? "-", controller?.deploymentName ?? "-", controller?.stateConfigMapName ?? "-", controller?.leaseName ?? "-", controller?.hostWorktreeMounted === true ? "mounted" : "not-mounted"]], ), "", table(["OBJECTS", "MANIFEST_SHA", "EXIT", "TIMED_OUT"], [[arrayRecords(payload.objects).length, shortSha(stringOrNull(payload.manifestSha256)), command?.exitCode ?? "-", command?.timedOut ?? "-"]]), command?.stderrTail ? `\nSTDERR\n${command.stderrTail}` : "", "", "NEXT", `status: ${next?.status ?? "-"}`, `dry-run: ${next?.dryRun ?? "-"}`, "", ].filter((line) => line !== "").join("\n"); } function renderStatusHuman(payload: Record, _options: ParsedOptions): string { const controller = asOptionalRecord(payload.controller); const followers = arrayRecords(payload.followers); const rows = followers.map((item) => { const source = asOptionalRecord(item.source); const target = asOptionalRecord(item.target); const budgets = asOptionalRecord(item.budgetSource); return [ item.id, item.phase, item.adapter, `${source?.branch ?? "-"}:${shortSha(stringOrNull(source?.observedSha))}`, shortSha(stringOrNull(target?.targetSha)), shortSha(stringOrNull(item.lastTriggeredSha)), shortSha(stringOrNull(item.lastSucceededSha)), item.pipelineRun ?? item.inFlightJob ?? "-", budgets?.endToEndSeconds ?? "-", item.message ?? "-", ]; }); const next = asOptionalRecord(payload.next); const errors = Array.isArray(payload.errors) ? payload.errors : []; const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48); return [ `CI/CD BRANCH-FOLLOWER STATUS (${payload.ok === false ? "degraded" : "ok"})`, "", table( ["CTRL_NS", "ROUTE", "DEPLOY", "READY", "PODS", "STATE_CM", "LEASE"], [[controller?.namespace ?? "-", controller?.route ?? "-", controller?.deploymentName ?? "-", `${controller?.availableReplicas ?? 0}/${controller?.replicas ?? 0}`, controller?.pods ?? "-", controller?.stateConfigMapPresent === true ? "present" : "missing", controller?.leaseHolder ?? "-"]], ), "", table(["FOLLOWER", "PHASE", "ADAPTER", "OBSERVED", "TARGET", "TRIGGERED", "SUCCEEDED", "IN_FLIGHT", "BUDGET", "MESSAGE"], rows), timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`, errors.length === 0 ? "" : `\nERRORS\n${errors.map((item) => `- ${item}`).join("\n")}`, "", "NEXT", `live-status: ${next?.liveStatus ?? "-"}`, `dry-run: ${next?.dryRun ?? "-"}`, "", ].filter((line) => line !== "").join("\n"); } function renderRunOnceHuman(payload: Record): string { const followers = arrayRecords(payload.followers); const stateWrites = arrayRecords(payload.stateWrites); const rows = followers.map((item) => { const source = asOptionalRecord(item.source); const target = asOptionalRecord(item.target); return [ item.id, item.phase, `${source?.branch ?? "-"}:${shortSha(stringOrNull(source?.observedSha))}`, shortSha(stringOrNull(target?.targetSha)), shortSha(stringOrNull(item.lastTriggeredSha)), item.inFlightJob ?? "-", item.decision ?? "-", ]; }); const next = asOptionalRecord(payload.next); const timingRows = followers.flatMap(timingRowsForFollower).slice(0, 48); const writeRows = stateWrites.map((item) => [ item.follower, item.ok === true ? "ok" : "failed", item.beforeResourceVersion ?? "-", item.afterResourceVersion ?? "-", item.preservedTiming === true ? "yes" : "no", item.exitCode ?? "-", item.message ?? "-", ]); return [ `CI/CD BRANCH-FOLLOWER RUN-ONCE (${payload.ok === false ? "blocked" : payload.dryRun === true ? "dry-run" : "ok"})`, "", table(["FOLLOWER", "PHASE", "OBSERVED", "TARGET", "TRIGGERED", "IN_FLIGHT", "DECISION"], rows), timingRows.length === 0 ? "" : `\nSTAGE TIMINGS\n${table(["FOLLOWER", "STAGE", "STATUS", "SECONDS", "BUDGET", "OBJECT"], timingRows)}`, writeRows.length === 0 ? "" : `\nSTATE WRITES\n${table(["FOLLOWER", "STATUS", "BEFORE_RV", "AFTER_RV", "PRESERVED", "EXIT", "MESSAGE"], writeRows)}`, "", "NEXT", `status: ${next?.status ?? "-"}`, `live-status: ${next?.liveStatus ?? "-"}`, "", ].join("\n"); } function timingRowsForFollower(item: Record): unknown[][] { const timings = asOptionalRecord(item.timings); if (timings === null) return []; const budget = numberOrNull(timings.budgetSeconds); const rows: unknown[][] = [[ item.id, "total", stringOrNull(timings.totalStatus) ?? "unknown", formatSeconds(numberOrNull(timings.totalSeconds)), formatSeconds(budget), [stringOrNull(timings.totalSource), shortSha(stringOrNull(timings.sourceCommit))].filter((value) => value !== null && value !== "-").join(":") || "-", ]]; for (const stage of arrayRecords(timings.stages)) { rows.push([ item.id, stage.stage, stage.status, formatSeconds(numberOrNull(stage.seconds)), formatSeconds(numberOrNull(stage.budgetSeconds)), stringOrNull(stage.object) ?? "-", ]); } return rows; } function formatSeconds(value: number | null): string { return value === null ? "-" : `${value}s`; } function renderCleanupStateHuman(payload: Record): string { const controller = asOptionalRecord(payload.controller); const command = asOptionalRecord(payload.command); const followers = arrayRecords(payload.followers); const next = asOptionalRecord(payload.next); const rows = followers.map((item) => [ item.id, item.statePresent === true ? "present" : "missing", item.cleanup ?? "-", ]); return [ `CI/CD BRANCH-FOLLOWER CLEANUP-STATE (${payload.ok === false ? "failed" : payload.dryRun === true ? "dry-run" : "ok"})`, "", table( ["NAMESPACE", "ROUTE", "STATE_CM", "STATE_CM_PRESENT"], [[controller?.namespace ?? "-", controller?.route ?? "-", controller?.stateConfigMapName ?? "-", payload.stateConfigMapPresent === true ? "present" : "missing"]], ), "", table(["FOLLOWER", "STATE", "CLEANUP"], rows), command === null ? "" : `\nPATCH\nexit=${command.exitCode ?? "-"} timedOut=${command.timedOut ?? "-"}`, "", "NEXT", `status: ${next?.status ?? "-"}`, `run-once: ${next?.runOnce ?? "-"}`, "", ].filter((line) => line !== "").join("\n"); } function renderDrillDownHuman(payload: Record): string { if (payload.follower === undefined) { const followers = arrayRecords(payload.followers); return [ `CI/CD BRANCH-FOLLOWER ${String(payload.action ?? "drill-down").toUpperCase()}`, "", table(["FOLLOWER", "ADAPTER", "STATUS_AUTHORITY"], followers.map((item) => [item.id, item.adapter, item.statusAuthority ?? "k8s-native"])), "", ].join("\n"); } const summary = asOptionalRecord(payload.summary); return [ `CI/CD BRANCH-FOLLOWER ${String(payload.action ?? "drill-down").toUpperCase()} (${payload.ok === false ? "failed" : "ok"})`, "", table( ["FOLLOWER", "ADAPTER", "AUTHORITY", "PHASE", "OBSERVED", "TARGET", "PIPELINERUN", "MESSAGE"], [[payload.follower, payload.adapter ?? "-", payload.statusAuthority ?? "k8s-native", summary?.phase ?? "-", shortSha(stringOrNull(summary?.observedSha)), shortSha(stringOrNull(summary?.targetSha)), summary?.pipelineRun ?? "-", summary?.message ?? "-"]], ), "", ].filter((line) => line !== "").join("\n"); } function arrayRecords(value: unknown): Record[] { return Array.isArray(value) ? value.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; } function arrayText(value: unknown): string { return Array.isArray(value) ? value.map(String).join(",") : "-"; } function table(headers: readonly string[], rows: readonly (readonly unknown[])[]): string { const normalized = rows.map((row) => headers.map((_, index) => cell(row[index]))); const widths = headers.map((header, index) => Math.max(header.length, ...normalized.map((row) => row[index]?.length ?? 0))); const format = (row: readonly string[]) => row.map((value, index) => value.padEnd(widths[index] ?? 0)).join(" ").trimEnd(); return [format(headers), format(headers.map((header) => "-".repeat(header.length))), ...normalized.map(format)].join("\n"); } function cell(value: unknown): string { if (value === null || value === undefined || value === "") return "-"; const text = String(value).replace(/\s+/gu, " "); return text.length > 96 ? `${text.slice(0, 93)}...` : text; }