From e9cc4f8ed32bc9cf973c3779bf378f73adf0faf4 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 3 Jul 2026 19:33:07 +0000 Subject: [PATCH] fix: flush follower mirror from thin cicd route --- .../references/branch-follower.md | 4 +- scripts/src/cicd-branch-follower.ts | 2583 +++++++++++++++ scripts/src/cicd-native-summary.ts | 230 ++ scripts/src/cicd.ts | 2768 +---------------- 4 files changed, 2822 insertions(+), 2763 deletions(-) create mode 100644 scripts/src/cicd-branch-follower.ts create mode 100644 scripts/src/cicd-native-summary.ts diff --git a/.agents/skills/unidesk-cicd/references/branch-follower.md b/.agents/skills/unidesk-cicd/references/branch-follower.md index c8e08eba..cb1bb68f 100644 --- a/.agents/skills/unidesk-cicd/references/branch-follower.md +++ b/.agents/skills/unidesk-cicd/references/branch-follower.md @@ -44,7 +44,7 @@ Follower-scoped commands such as `status --follower`, `events --follower`, `logs Multi-follower status summaries should omit per-follower `command.payload`/native drill-down payloads entirely; those belong to follower-scoped `events`/`logs`/`debug-step` queries. Default all-follower status must remain parseable below the transport stdout limit. -`scripts/src/cicd.ts` should stay a thin branch-follower route/orchestration entry, not a catch-all implementation file. Rendering, debug steps, controller manifests, native K8s helpers, adapter-specific trigger/status logic and large data compactors must live in responsibility-specific modules before `cicd.ts` approaches the 3000-line hard split point. +`scripts/src/cicd.ts` must stay a thin top-level CI/CD route entry. Branch-follower implementation belongs in `scripts/src/cicd-branch-follower.ts` and responsibility-specific modules; rendering, debug steps, controller manifests, native K8s helpers, adapter-specific trigger/status logic and large data compactors must be split before any implementation file approaches the 3000-line hard split point. `status-read`, `events`, `logs` and debug summaries must expose compact closeout gate details when a follower is not aligned: git-mirror readiness, Tekton PipelineRun condition, Argo sync/health, runtime target sha/readiness and short errors. Repeating only phase/observed/target/message is a visibility defect and must be fixed before further rollout tuning. @@ -56,6 +56,8 @@ When Argo exposes operation start/finish timestamps, stage timing rows should re The automatic controller loop is non-blocking, so closeout acceleration cannot live only in the user-facing `--wait` path. Once a triggered PipelineRun has succeeded and required runtime/GitOps gates are not aligned, the in-cluster controller path should perform the same bounded target-side Argo refresh used by wait closeout; otherwise convergence depends on Argo's background poll interval and can exceed the 120s budget even when Tekton finished quickly. +The same rule applies to git-mirror post-flush. If native status shows runtime/Argo are aligned but GitOps mirror is still pending flush, the automatic controller loop must run the bounded target-side git-mirror flush instead of leaving a follower in `ClosingOut` until a manual wait/closeout path is used. + Stage timing rows must not label optional gates as `not-ready` when they are not part of that follower's closeout contract. For sentinel-like followers without a GitOps branch flush gate, git-mirror source snapshot readiness should render as source-ready/ready, while missing GitOps `githubInSync` remains `-`/not-applicable instead of a failure-looking state. ## Source Authority diff --git a/scripts/src/cicd-branch-follower.ts b/scripts/src/cicd-branch-follower.ts new file mode 100644 index 00000000..76367b18 --- /dev/null +++ b/scripts/src/cicd-branch-follower.ts @@ -0,0 +1,2583 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower. +// Responsibility: YAML-first K8s branch-follower controller, status, and adapter orchestration. +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { isAbsolute } from "node:path"; +import { repoRoot, rootPath, type UniDeskConfig } from "./config"; +import { runCommand, type CommandResult } from "./command"; +import { startJob } from "./jobs"; +import type { RenderedCliResult } from "./output"; +import { renderMachine, renderResult } from "./cicd-render"; +import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; +import { agentRunImageArtifact, renderAgentRunGitopsFiles } from "./agentrun-manifests"; +import { agentRunPipelineRunName, resolveAgentRunLaneTarget } from "./agentrun-lanes"; +import { yamlLaneGitMirrorJobManifest, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLanePipelineRunManifest } from "./agentrun/secrets"; +import { yamlLaneK3sBuildImageJobManifest } from "./agentrun/yaml-lane"; +import { nodeRuntimePipelineRunName } from "./hwlab-node/cleanup"; +import { nodeRuntimeGitMirrorJobManifest } from "./hwlab-node/render"; +import { nodeRuntimeGitMirrorTarget, nodeRuntimePipelineRunManifest } from "./hwlab-node/web-probe"; +import { loadSentinelCicdState } from "./hwlab-node-web-sentinel-cicd"; +import { sentinelPublishPipelineRunManifest } from "./hwlab-node-web-sentinel-cicd-jobs"; +import { sentinelPipelineRunName } from "./hwlab-node-web-sentinel-cicd-shared"; +import { transPath } from "./hwlab-node/runtime-common"; +import { configRefGraph, resolveConfigRefString } from "./ops/config-refs"; +import { renderControllerManifests, renderControllerReconcileJob, waitForJobShell } from "./cicd-controller-render"; +import { buildDebugStep } from "./cicd-debug"; +import { runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh"; +import { nativeCicdScriptLoadShell, readNativeObjectBundle } from "./cicd-native-bundle"; +import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native"; +import { argoApplicationReady, nativeArgoSummary, nativeGitMirrorReady, nativeGitMirrorRequired, nativeGitMirrorSummary, nativePipelineRunSummary, nativeRuntimeSummary, pipelineRunSucceeded, runtimeTargetShaFromWorkloads, runtimeWorkloadsReady } from "./cicd-native-summary"; +import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types"; +import { + arrayField, + asRecord, + booleanField, + integerField, + readYamlRecord, + recordField, + redactText, + shQuote, + stringArrayField, + stringField, +} from "./platform-infra-ops-library"; + +const DEFAULT_CONFIG_PATH = "config/cicd-branch-followers.yaml"; +const SPEC_REF = "PJ2026-01060703"; +const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower"; + +export function cicdHelp(): unknown { + return { + command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs", + output: "text by default; use --json, --raw, or -o json|yaml for machine output", + usage: [ + "bun scripts/cli.ts cicd branch-follower plan", + "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", + "bun scripts/cli.ts cicd branch-follower status", + "bun scripts/cli.ts cicd branch-follower status --live", + "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", + "bun scripts/cli.ts cicd branch-follower run-once --follower hwlab-jd01-v03 --confirm --wait", + "bun scripts/cli.ts cicd branch-follower debug-step --follower web-probe-sentinel-master --step state-read", + "bun scripts/cli.ts cicd branch-follower debug-step --follower web-probe-sentinel-master --step state-write --confirm", + "bun scripts/cli.ts cicd branch-follower cleanup-state --follower web-probe-sentinel-master --confirm", + "bun scripts/cli.ts cicd branch-follower events --follower agentrun-jd01-v02", + "bun scripts/cli.ts cicd branch-follower logs --follower web-probe-sentinel-master", + ], + config: DEFAULT_CONFIG_PATH, + spec: `${SPEC_REF} ${SPEC_VERSION}`, + description: "Deploy and inspect the YAML-first Kubernetes branch follower that follows HWLAB v0.3, AgentRun v0.2, and the selected web-probe sentinel master lane without using host worktrees as source authority.", + }; +} + +export async function runCicdCommand(_config: UniDeskConfig | null, args: string[]): Promise { + const top = args[0]; + if (top === undefined || isHelpToken(top)) return renderMachine("cicd", cicdHelp(), "json"); + if (top !== "branch-follower") { + throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs"); + } + const options = parseOptions(args.slice(1)); + const command = commandLabel(options); + if (options.action === "help") return renderMachine(command, cicdHelp(), "json"); + const registry = readRegistry(options.configPath); + switch (options.action) { + case "plan": + return renderResult(command, buildPlan(registry, options), options); + case "apply": + return renderResult(command, await applyController(registry, options), options); + case "status": + return renderResult(command, await buildStatus(registry, options), options); + case "run-once": + return renderResult(command, await runOnce(registry, options), options); + case "debug-step": + return renderResult(command, await buildDebugStep(registry, options, { selectFollowers, readK8sState, readAdapterStatus, decideAndMaybeTrigger, writeFollowerState, runKubeScript }), options); + case "cleanup-state": + return renderResult(command, cleanupState(registry, options), options); + case "events": + case "logs": + return renderResult(command, await runFollowerDrillDown(registry, options), options); + case "help": + return renderMachine(command, cicdHelp(), "json"); + } +} + +function parseOptions(args: string[]): ParsedOptions { + const actionToken = args[0]; + if (actionToken === undefined || isHelpToken(actionToken)) { + return defaultOptions("help", args.slice(actionToken === undefined ? 0 : 1)); + } + if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs"].includes(actionToken)) { + throw new Error(`cicd branch-follower unknown action: ${actionToken}`); + } + const action = actionToken as BranchFollowerAction; + const options = defaultOptions(action, []); + const rest = args.slice(1); + for (let index = 0; index < rest.length; index += 1) { + const arg = rest[index] ?? ""; + if (isHelpToken(arg)) { + options.action = "help"; + } else if (arg === "--config") { + options.configPath = valueOption(rest, ++index, arg); + } else if (arg === "--follower" || arg === "--target") { + options.followerId = simpleId(valueOption(rest, ++index, arg), arg); + } else if (arg === "--all") { + options.all = true; + } else if (arg === "--confirm") { + options.confirm = true; + } else if (arg === "--dry-run") { + options.dryRun = true; + } else if (arg === "--wait") { + options.wait = true; + } else if (arg === "--in-cluster") { + options.inCluster = true; + } else if (arg === "--controller") { + if (isInClusterRuntime()) options.inCluster = true; + else if (options.action === "status") options.live = true; + } else if (arg === "--live") { + options.live = true; + } else if (arg === "--no-live") { + options.noLive = true; + } else if (arg === "--full") { + options.full = true; + } else if (arg === "--raw" || arg === "--json") { + options.raw = true; + options.full = true; + options.output = "json"; + } else if (arg === "--record-state") { + options.recordState = true; + } else if (arg === "--step") { + options.debugStep = debugStepOption(valueOption(rest, ++index, arg)); + } else if (arg === "-o" || arg === "--output") { + const value = valueOption(rest, ++index, arg); + if (value !== "json" && value !== "yaml" && value !== "wide" && value !== "text") throw new Error(`${arg} must be json, yaml, wide, or text`); + options.output = value === "wide" || value === "text" ? "human" : value; + if (value === "json" || value === "yaml") { + options.raw = true; + options.full = true; + } + } else if (arg === "--limit") { + options.limit = positiveInt(valueOption(rest, ++index, arg), arg); + } else if (arg === "--tail-bytes" || arg === "--tail") { + options.tailBytes = positiveInt(valueOption(rest, ++index, arg), arg); + } else if (arg === "--timeout-seconds") { + options.timeoutSeconds = positiveInt(valueOption(rest, ++index, arg), arg); + } else { + throw new Error(`unsupported cicd branch-follower option: ${arg}`); + } + } + if (options.confirm && options.dryRun) throw new Error("cicd branch-follower accepts only one of --confirm or --dry-run"); + if (options.action === "apply" && !options.confirm) options.dryRun = true; + if (options.action === "run-once" && !options.confirm) options.dryRun = true; + if (options.action === "debug-step" && !options.confirm) options.dryRun = true; + if (options.action === "cleanup-state" && !options.confirm) options.dryRun = true; + if (options.action === "run-once" && options.confirm && !options.all && options.followerId === null) { + throw new Error("run-once --confirm requires --all or --follower "); + } + if (options.action === "cleanup-state" && options.confirm && !options.all && options.followerId === null) { + throw new Error("cleanup-state --confirm requires --all or --follower "); + } + if (options.action === "debug-step" && options.followerId === null) { + throw new Error("debug-step requires --follower "); + } + return options; +} + +function debugStepOption(value: string): BranchFollowerDebugStep { + if (value === "state-read" || value === "status-read" || value === "decide" || value === "state-write") return value; + throw new Error("--step must be state-read, status-read, decide, or state-write"); +} + +function isInClusterRuntime(): boolean { + return Boolean(process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT); +} + +function defaultOptions(action: BranchFollowerAction, _args: string[]): ParsedOptions { + return { + action, + configPath: DEFAULT_CONFIG_PATH, + followerId: null, + all: false, + confirm: false, + dryRun: false, + wait: false, + inCluster: false, + live: false, + noLive: false, + full: false, + raw: false, + recordState: false, + debugStep: null, + output: "human", + limit: 20, + tailBytes: 12000, + timeoutSeconds: null, + }; +} + +function valueOption(args: string[], index: number, option: string): string { + const value = args[index]; + if (value === undefined || value.length === 0 || value.startsWith("--")) throw new Error(`${option} requires a value`); + return value; +} + +function simpleId(value: string, option: string): string { + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${option} must be a simple id`); + return value; +} + +function positiveInt(value: string, option: string): number { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${option} must be a positive integer`); + return parsed; +} + +function isHelpToken(value: string): boolean { + return value === "-h" || value === "--help" || value === "help"; +} + +function readRegistry(configPath: string): BranchFollowerRegistry { + const absolute = isAbsolute(configPath) ? configPath : rootPath(configPath); + const rawText = readFileSync(absolute, "utf8"); + const root = readYamlRecord>(absolute, "CicdBranchFollowerRegistry"); + const metadata = recordField(root, "metadata", configPath); + const controller = parseController(recordField(root, "controller", configPath)); + const followers = arrayField(root, "followers", configPath).map(parseFollower); + const ids = new Set(); + for (const follower of followers) { + if (ids.has(follower.id)) throw new Error(`${configPath}.followers has duplicate id ${follower.id}`); + ids.add(follower.id); + } + return { + path: configPath, + rawText, + rawSha256: createHash("sha256").update(rawText).digest("hex"), + metadata: { + id: stringField(metadata, "id", `${configPath}.metadata`), + owner: stringField(metadata, "owner", `${configPath}.metadata`), + specRef: stringField(metadata, "specRef", `${configPath}.metadata`), + version: stringField(metadata, "version", `${configPath}.metadata`), + }, + controller, + followers, + }; +} + +function parseController(root: Record): ControllerSpec { + const source = recordField(root, "source", "controller"); + const authority = recordField(source, "sourceAuthority", "controller.source"); + const snapshot = recordField(source, "sourceSnapshot", "controller.source"); + const loop = recordField(root, "loop", "controller"); + const budgets = recordField(root, "budgets", "controller"); + const result: ControllerSpec = { + namespace: stringField(root, "namespace", "controller"), + kubeRoute: stringField(root, "kubeRoute", "controller"), + fieldManager: stringField(root, "fieldManager", "controller"), + serviceAccountName: stringField(root, "serviceAccountName", "controller"), + deploymentName: stringField(root, "deploymentName", "controller"), + configMapName: stringField(root, "configMapName", "controller"), + stateConfigMapName: stringField(root, "stateConfigMapName", "controller"), + leaseName: stringField(root, "leaseName", "controller"), + image: stringField(root, "image", "controller"), + labels: stringMap(recordField(root, "labels", "controller"), "controller.labels"), + source: { + repository: stringField(source, "repository", "controller.source"), + branch: stringField(source, "branch", "controller.source"), + gitMirrorReadUrl: stringField(source, "gitMirrorReadUrl", "controller.source"), + gitMirrorCachePvcName: stringField(source, "gitMirrorCachePvcName", "controller.source"), + githubSsh: parseControllerGithubSsh(recordField(source, "githubSsh", "controller.source")), + sourceAuthority: { + mode: stringField(authority, "mode", "controller.source.sourceAuthority"), + resolver: stringField(authority, "resolver", "controller.source.sourceAuthority"), + allowHostGit: booleanField(authority, "allowHostGit", "controller.source.sourceAuthority"), + allowHostWorkspace: booleanField(authority, "allowHostWorkspace", "controller.source.sourceAuthority"), + allowGithubDirectInPipeline: booleanField(authority, "allowGithubDirectInPipeline", "controller.source.sourceAuthority"), + }, + sourceSnapshot: { + stageRefPrefix: stringField(snapshot, "stageRefPrefix", "controller.source.sourceSnapshot"), + missingObjectPolicy: stringField(snapshot, "missingObjectPolicy", "controller.source.sourceSnapshot"), + refreshPolicy: stringField(snapshot, "refreshPolicy", "controller.source.sourceSnapshot"), + }, + }, + loop: { + intervalSeconds: integerField(loop, "intervalSeconds", "controller.loop"), + reconcileTimeoutSeconds: integerField(loop, "reconcileTimeoutSeconds", "controller.loop"), + leaseDurationSeconds: integerField(loop, "leaseDurationSeconds", "controller.loop"), + terminationGracePeriodSeconds: integerField(loop, "terminationGracePeriodSeconds", "controller.loop"), + }, + budgets: { + applyWaitSeconds: integerField(budgets, "applyWaitSeconds", "controller.budgets"), + statusSeconds: integerField(budgets, "statusSeconds", "controller.budgets"), + runOnceSeconds: integerField(budgets, "runOnceSeconds", "controller.budgets"), + reconcileJobTtlSeconds: integerField(budgets, "reconcileJobTtlSeconds", "controller.budgets"), + reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"), + reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"), + reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"), + nativeTransportGraceSeconds: integerField(budgets, "nativeTransportGraceSeconds", "controller.budgets"), + nativePollIntervalSeconds: integerField(budgets, "nativePollIntervalSeconds", "controller.budgets"), + }, + }; + if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) { + throw new Error("controller.source.sourceAuthority must disable host git, host workspace, and direct GitHub pipeline fallback"); + } + return result; +} + +function parseControllerGithubSsh(root: Record): ControllerSpec["source"]["githubSsh"] { + return { + secretName: stringField(root, "secretName", "controller.source.githubSsh"), + privateKeySecretKey: stringField(root, "privateKeySecretKey", "controller.source.githubSsh"), + proxyHost: stringField(root, "proxyHost", "controller.source.githubSsh"), + proxyPort: integerField(root, "proxyPort", "controller.source.githubSsh"), + }; +} + +function parseFollower(root: Record, index: number): FollowerSpec { + const label = `followers[${index}]`; + const source = recordField(root, "source", label); + const target = recordField(root, "target", label); + const budgets = recordField(root, "budgets", label); + const commands = recordField(root, "commands", label); + const nativeStatus = recordField(root, "nativeStatus", label); + const closeout = recordField(root, "closeout", label); + const configRefs = stringMap(recordField(target, "configRefs", `${label}.target`), `${label}.target.configRefs`); + return { + id: simpleId(stringField(root, "id", label), `${label}.id`), + enabled: booleanField(root, "enabled", label), + adapter: stringField(root, "adapter", label), + description: stringField(root, "description", label), + source: { + repository: stringField(source, "repository", `${label}.source`), + branch: stringField(source, "branch", `${label}.source`), + branchRef: stringField(source, "branchRef", `${label}.source`), + authorityRef: stringField(source, "authorityRef", `${label}.source`), + snapshotPrefix: stringField(source, "snapshotPrefix", `${label}.source`), + snapshotRef: stringField(source, "snapshotRef", `${label}.source`), + }, + target: { + node: stringField(target, "node", `${label}.target`), + lane: stringField(target, "lane", `${label}.target`), + namespace: stringField(target, "namespace", `${label}.target`), + sentinel: typeof target.sentinel === "string" && target.sentinel.length > 0 ? target.sentinel : null, + configRefs, + }, + budgets: { + endToEndSeconds: integerField(budgets, "endToEndSeconds", `${label}.budgets`), + statusSeconds: integerField(budgets, "statusSeconds", `${label}.budgets`), + triggerSeconds: integerField(budgets, "triggerSeconds", `${label}.budgets`), + sourceSyncSeconds: integerField(budgets, "sourceSyncSeconds", `${label}.budgets`), + controlPlaneRefreshSeconds: integerField(budgets, "controlPlaneRefreshSeconds", `${label}.budgets`), + capabilityJobTtlSeconds: integerField(budgets, "capabilityJobTtlSeconds", `${label}.budgets`), + capabilityJobBackoffLimit: integerField(budgets, "capabilityJobBackoffLimit", `${label}.budgets`), + }, + commands: { + plan: parseCommand(recordField(commands, "plan", `${label}.commands`), `${label}.commands.plan`), + status: parseCommand(recordField(commands, "status", `${label}.commands`), `${label}.commands.status`), + trigger: parseCommand(recordField(commands, "trigger", `${label}.commands`), `${label}.commands.trigger`), + events: parseCommand(recordField(commands, "events", `${label}.commands`), `${label}.commands.events`), + logs: parseCommand(recordField(commands, "logs", `${label}.commands`), `${label}.commands.logs`), + }, + nativeStatus: parseNativeStatus(nativeStatus, `${label}.nativeStatus`), + closeoutChecks: stringArrayField(closeout, "checks", `${label}.closeout`), + }; +} + +function parseNativeStatus(root: Record, label: string): NativeStatusSpec { + const source = recordField(root, "source", label); + const tekton = asOptionalRecord(root.tekton); + const argo = asOptionalRecord(root.argo); + const runtime = asOptionalRecord(root.runtime); + return { + source: { + gitMirrorReadUrl: stringField(source, "gitMirrorReadUrl", `${label}.source`), + gitMirrorNamespace: stringField(source, "gitMirrorNamespace", `${label}.source`), + gitMirrorDeployment: stringField(source, "gitMirrorDeployment", `${label}.source`), + repoPath: stringField(source, "repoPath", `${label}.source`), + }, + tekton: tekton === null + ? null + : { + namespace: stringField(tekton, "namespace", `${label}.tekton`), + pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${label}.tekton`), + }, + argo: argo === null + ? null + : { + namespace: stringField(argo, "namespace", `${label}.argo`), + application: stringField(argo, "application", `${label}.argo`), + }, + runtime: runtime === null + ? null + : { + namespace: stringField(runtime, "namespace", `${label}.runtime`), + workloads: arrayField(runtime, "workloads", `${label}.runtime`).map((item, index) => parseNativeWorkload(item, `${label}.runtime.workloads[${index}]`)), + }, + }; +} + +function parseNativeWorkload(value: unknown, label: string): NativeWorkloadSpec { + const root = asRecord(value, label); + const kind = stringField(root, "kind", label); + if (kind !== "Deployment" && kind !== "StatefulSet") throw new Error(`${label}.kind must be Deployment or StatefulSet`); + const sourceCommit = asOptionalRecord(root.sourceCommit) ?? {}; + return { + kind, + name: stringField(root, "name", label), + sourceCommit: { + labels: optionalStringArrayField(sourceCommit, "labels", `${label}.sourceCommit`), + annotations: optionalStringArrayField(sourceCommit, "annotations", `${label}.sourceCommit`), + podLabels: optionalStringArrayField(sourceCommit, "podLabels", `${label}.sourceCommit`), + podAnnotations: optionalStringArrayField(sourceCommit, "podAnnotations", `${label}.sourceCommit`), + env: optionalStringArrayField(sourceCommit, "env", `${label}.sourceCommit`), + }, + }; +} + +function parseCommand(root: Record, label: string): CommandSpec { + return { + argv: stringArrayField(root, "argv", label), + timeoutSeconds: integerField(root, "timeoutSeconds", label), + }; +} + +function optionalStringArrayField(root: Record, key: string, label: string): string[] { + return root[key] === undefined ? [] : stringArrayField(root, key, label); +} + +function stringMap(root: Record, label: string): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(root)) { + if (typeof value !== "string" || value.length === 0) throw new Error(`${label}.${key} must be a non-empty string`); + result[key] = value; + } + return result; +} + +function buildPlan(registry: BranchFollowerRegistry, options: ParsedOptions): Record { + const selected = selectFollowers(registry, options, { includeDisabled: true }); + const followers = selected.map((follower) => { + const branchValue = safeResolveString(follower.source.branchRef); + const graph = configRefGraph([ + { id: "source.branch", ref: follower.source.branchRef }, + { id: "source.authority", ref: follower.source.authorityRef }, + { id: "source.snapshot", ref: follower.source.snapshotRef }, + ...Object.entries(follower.target.configRefs).map(([id, ref]) => ({ id: `target.${id}`, ref })), + ]); + const warnings: string[] = []; + if (branchValue !== null && branchValue !== follower.source.branch) warnings.push(`source.branch ${follower.source.branch} differs from ${follower.source.branchRef} value ${branchValue}`); + return { + id: follower.id, + enabled: follower.enabled, + adapter: follower.adapter, + description: follower.description, + source: { + repository: follower.source.repository, + branch: follower.source.branch, + branchRef: follower.source.branchRef, + resolvedBranch: branchValue, + snapshotPrefix: follower.source.snapshotPrefix, + }, + target: follower.target, + budgets: follower.budgets, + commands: redactCommands(follower), + nativeStatus: nativeStatusPlan(follower.nativeStatus), + closeoutChecks: follower.closeoutChecks, + configRefGraph: graph, + warnings, + }; + }); + return { + ok: true, + action: "plan", + spec: `${SPEC_REF} ${SPEC_VERSION}`, + registry: registrySummary(registry), + hostWorktreeAuthority: false, + sourceAuthority: { + mode: registry.controller.source.sourceAuthority.mode, + resolver: registry.controller.source.sourceAuthority.resolver, + allowHostGit: registry.controller.source.sourceAuthority.allowHostGit, + allowHostWorkspace: registry.controller.source.sourceAuthority.allowHostWorkspace, + allowGithubDirectInPipeline: registry.controller.source.sourceAuthority.allowGithubDirectInPipeline, + }, + controller: { + namespace: registry.controller.namespace, + kubeRoute: registry.controller.kubeRoute, + deploymentName: registry.controller.deploymentName, + stateConfigMapName: registry.controller.stateConfigMapName, + leaseName: registry.controller.leaseName, + image: registry.controller.image, + loop: registry.controller.loop, + budgets: registry.controller.budgets, + }, + followers, + next: { + apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", + status: "bun scripts/cli.ts cicd branch-follower status", + dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", + }, + }; +} + +async function applyController(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { + const manifests = renderControllerManifests(registry); + const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; + const manifestBase64 = Buffer.from(manifestYaml, "utf8").toString("base64"); + const waitSeconds = options.timeoutSeconds ?? registry.controller.budgets.applyWaitSeconds; + const script = [ + "set -eu", + "tmp=$(mktemp)", + "base64 -d >\"$tmp\" <<'UNIDESK_CICD_BRANCH_FOLLOWER_MANIFEST_B64'", + manifestBase64, + "UNIDESK_CICD_BRANCH_FOLLOWER_MANIFEST_B64", + options.dryRun + ? `kubectl apply --dry-run=server --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp"` + : `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp"`, + !options.dryRun && options.wait + ? `kubectl -n ${shQuote(registry.controller.namespace)} rollout status deploy/${shQuote(registry.controller.deploymentName)} --timeout=${waitSeconds}s` + : "true", + `kubectl -n ${shQuote(registry.controller.namespace)} get deploy/${shQuote(registry.controller.deploymentName)} cm/${shQuote(registry.controller.configMapName)} cm/${shQuote(registry.controller.stateConfigMapName)} lease/${shQuote(registry.controller.leaseName)} -o wide 2>/dev/null || true`, + ].join("\n"); + const result = runKubeScript(registry, options, script, "", (waitSeconds + 15) * 1000); + return { + ok: result.exitCode === 0, + action: "apply", + dryRun: options.dryRun, + wait: options.wait, + registry: registrySummary(registry), + objects: manifests.map((item) => objectRef(item)), + manifestSha256: createHash("sha256").update(manifestYaml).digest("hex"), + controller: { + namespace: registry.controller.namespace, + route: registry.controller.kubeRoute, + deploymentName: registry.controller.deploymentName, + stateConfigMapName: registry.controller.stateConfigMapName, + leaseName: registry.controller.leaseName, + hostWorktreeMounted: false, + sourceMode: "k8s-git-mirror-to-emptyDir", + }, + command: commandCompact(result, options), + next: { + status: "bun scripts/cli.ts cicd branch-follower status", + logs: `bun scripts/cli.ts cicd branch-follower logs --follower ${registry.followers[0]?.id ?? ""}`, + dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", + }, + }; +} + +async function buildStatus(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { + let k8s = readK8sState(registry, options); + const wantsLive = options.live || (!options.noLive && Object.keys(k8s.stateByFollower).length === 0); + const refresh = wantsLive && !options.inCluster ? runControllerReconcileJob(registry, options, { dryRun: true, wait: true, recordState: true }) : null; + if (refresh !== null) k8s = readK8sState(registry, options); + const shouldLive = wantsLive && options.inCluster; + const selected = selectFollowers(registry, options, { includeDisabled: true }); + const followers = []; + for (const follower of selected) { + const stored = k8s.stateByFollower[follower.id] ?? {}; + const live = shouldLive && follower.enabled ? await readAdapterStatus(registry, follower, options) : null; + followers.push(mergeFollowerStatus(registry, follower, stored, live, shouldLive)); + } + return { + ok: k8s.ok && followers.every((item) => item.ok !== false), + action: "status", + live: shouldLive, + registry: registrySummary(registry), + controller: controllerStatusSummary(registry, k8s), + followers, + refresh, + errors: k8s.errors, + next: { + apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", + liveStatus: "bun scripts/cli.ts cicd branch-follower status --live", + dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", + }, + }; +} + +async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { + if (!options.inCluster) { + const refresh = runControllerReconcileJob(registry, options, { dryRun: options.dryRun, wait: true, recordState: true }); + const k8s = readK8sState(registry, options); + const selected = selectFollowers(registry, options, { includeDisabled: false }); + return { + ok: refresh.ok, + action: "run-once", + dryRun: options.dryRun, + confirm: options.confirm, + wait: true, + controller: false, + execution: "k8s-native-reconcile-job", + registry: registrySummary(registry), + job: refresh, + followers: selected.map((follower) => mergeFollowerStatus(registry, follower, k8s.stateByFollower[follower.id] ?? {}, null, false)), + warnings: refresh.ok ? [] : [`reconcile job failed: ${refresh.message}`], + next: { + status: "bun scripts/cli.ts cicd branch-follower status", + liveStatus: "bun scripts/cli.ts cicd branch-follower status --live", + }, + }; + } + const selected = selectFollowers(registry, options, { includeDisabled: false }); + const previous = readK8sState(registry, options); + const results: FollowerState[] = []; + const stateWriteWarnings: string[] = []; + const stateWrites: Record[] = []; + for (const follower of selected) { + const oldState = previous.stateByFollower[follower.id] ?? {}; + const live = await readAdapterStatus(registry, follower, options); + const state = await decideAndMaybeTrigger(registry, follower, oldState, live, options); + if (!options.dryRun || options.recordState) { + const write = writeFollowerState(registry, state, options); + const writeSummary = stateWriteSummary(follower.id, write); + stateWrites.push(writeSummary); + if (write.exitCode !== 0) { + const warning = `state write failed for ${follower.id}: ${tailText(write.stderr || write.stdout, 300)}`; + state.warnings.push(warning); + stateWriteWarnings.push(warning); + } + } + results.push(state); + } + return { + ok: results.every((item) => item.phase !== "Failed" && item.phase !== "Blocked"), + action: "run-once", + dryRun: options.dryRun, + confirm: options.confirm, + wait: options.wait, + controller: options.inCluster, + registry: registrySummary(registry), + followers: results, + stateWrites, + warnings: stateWriteWarnings, + next: { + status: "bun scripts/cli.ts cicd branch-follower status", + liveStatus: "bun scripts/cli.ts cicd branch-follower status --live", + }, + }; +} + +function cleanupState(registry: BranchFollowerRegistry, options: ParsedOptions): Record { + const selected = selectFollowers(registry, options, { includeDisabled: true }); + const ids = selected.map((follower) => follower.id); + const before = kubeConfigMapFollowerState(registry, options); + const presentKeys = ids.filter((id) => before.stateByFollower[id] !== undefined); + const command = options.confirm ? removeFollowerStateKeys(registry, options, ids) : null; + return { + ok: command === null ? before.ok : command.exitCode === 0, + action: "cleanup-state", + dryRun: !options.confirm, + confirm: options.confirm, + execution: "k8s-state-configmap-key-delete", + registry: registrySummary(registry), + controller: { + namespace: registry.controller.namespace, + route: registry.controller.kubeRoute, + stateConfigMapName: registry.controller.stateConfigMapName, + }, + followers: ids.map((id) => ({ + id, + statePresent: before.stateByFollower[id] !== undefined, + cleanup: options.confirm ? "deleted-if-present" : "would-delete-if-present", + })), + stateConfigMapPresent: before.present, + presentKeys, + parsedDownstreamCliOutput: false, + command: command === null ? null : commandCompact(command, options), + errors: before.ok ? [] : [before.error], + next: { + status: "bun scripts/cli.ts cicd branch-follower status", + runOnce: options.followerId === null + ? "bun scripts/cli.ts cicd branch-follower run-once --all --confirm --wait" + : `bun scripts/cli.ts cicd branch-follower run-once --follower ${options.followerId} --confirm --wait`, + }, + }; +} + +async function runFollowerDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { + if (options.followerId === null) { + return { + ok: true, + action: options.action, + message: "select one follower to inspect native Kubernetes status objects", + followers: registry.followers.map((follower) => ({ + id: follower.id, + adapter: follower.adapter, + statusAuthority: "k8s-native", + nativeStatus: nativeStatusPlan(follower.nativeStatus), + })), + }; + } + const follower = registry.followers.find((item) => item.id === options.followerId); + if (follower === undefined) throw new Error(`unknown follower ${options.followerId}`); + if (!options.inCluster) { + const refresh = runControllerReconcileJob(registry, options, { dryRun: true, wait: true, recordState: true }); + const k8s = readK8sState(registry, options); + const stored = k8s.stateByFollower[follower.id] ?? {}; + const storedSource = asOptionalRecord(stored.source); + const storedTarget = asOptionalRecord(stored.target); + const command = asOptionalRecord(stored.command); + const native = asOptionalRecord(command?.payload); + return { + ok: refresh.ok && k8s.ok && Object.keys(stored).length > 0, + action: options.action, + follower: follower.id, + adapter: follower.adapter, + statusAuthority: "k8s-native-state-configmap", + parsedDownstreamCliOutput: false, + summary: { + phase: stringOrNull(stored.phase) ?? "Observed", + observedSha: stringOrNull(storedSource?.observedSha), + targetSha: stringOrNull(storedTarget?.targetSha), + pipelineRun: stringOrNull(stored.pipelineRun) ?? stringOrNull(stored.inFlightJob), + aligned: null, + message: stringOrNull(stored.decision) ?? "no controller state yet", + }, + native, + refresh, + errors: k8s.errors, + next: { + status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, + liveStatus: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --live`, + runOnceDryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`, + }, + }; + } + const live = await readAdapterStatus(registry, follower, options); + return { + ok: live.ok, + action: options.action, + follower: follower.id, + adapter: follower.adapter, + statusAuthority: "k8s-native", + parsedDownstreamCliOutput: false, + summary: { + phase: live.phase, + observedSha: live.observedSha, + targetSha: live.targetSha, + pipelineRun: live.pipelineRun, + aligned: live.aligned, + message: live.message, + }, + native: live.payload, + next: { + status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, + runOnceDryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`, + }, + }; +} + +async function decideAndMaybeTrigger( + registry: BranchFollowerRegistry, + follower: FollowerSpec, + previous: Record, + live: AdapterSummary, + options: ParsedOptions, +): Promise { + const warnings: string[] = []; + if (!live.ok) warnings.push(`status command failed: exitCode=${live.exitCode}${live.timedOut ? " timedOut=true" : ""}`); + const observedSha = live.observedSha; + let targetSha = live.targetSha; + const previousPhase = stringOrNull(previous.phase); + const previousLastTriggered = previousPhase === "Failed" ? null : stringOrNull(previous.lastTriggeredSha); + const previousInFlight = stringOrNull(previous.inFlightJob); + const previousLastSucceeded = stringOrNull(previous.lastSucceededSha); + const previousObserved = stringOrNull(recordAt(previous, ["source"])?.observedSha); + const previousTarget = stringOrNull(recordAt(previous, ["target"])?.targetSha); + const superseded = previousInFlight !== null && previousObserved !== null && observedSha !== null && previousObserved !== observedSha; + const nativePipelineRunMatchesObserved = observedSha !== null + && live.pipelineRunPresent === true + && live.pipelineRun !== null + && live.pipelineRun === expectedPipelineRunName(follower, observedSha); + let phase: BranchFollowerPhase; + let decision: string; + let triggerCommand: Record | undefined; + let inFlightJob: string | null = live.inFlightJob; + let lastTriggeredSha = live.lastTriggeredSha ?? previousLastTriggered ?? (nativePipelineRunMatchesObserved ? observedSha : null); + let lastSucceededSha = live.lastSucceededSha ?? previousLastSucceeded; + if (targetSha === null && observedSha !== null && previousLastSucceeded === observedSha && previousTarget === observedSha) targetSha = observedSha; + + if (!follower.enabled) { + phase = "Skipped"; + decision = "follower disabled"; + } else if (observedSha === null) { + phase = live.ok ? "Observed" : "Blocked"; + decision = "status did not expose an observed source sha; adapter trigger-current remains the dedupe authority"; + } else if (superseded) { + phase = "Superseded"; + decision = `previous in-flight sha ${shortSha(previousObserved)} was superseded by ${shortSha(observedSha)}`; + } else if (targetSha !== null && targetSha === observedSha && live.aligned === true) { + phase = "Noop"; + decision = "target already matches observed source sha"; + inFlightJob = null; + lastSucceededSha = observedSha; + } else if (lastTriggeredSha !== null && lastTriggeredSha === observedSha && live.pipelineRunPresent === true) { + phase = "ClosingOut"; + decision = targetSha === observedSha + ? "target sha matches observed source sha but native closeout gates are not complete" + : "same sha was already triggered; use status/events/logs for closeout"; + } else if (lastTriggeredSha !== null && lastTriggeredSha === observedSha) { + phase = "PendingTrigger"; + decision = "same sha has a trigger record but no PipelineRun is present; retrying trigger"; + } else if (targetSha !== null && targetSha === observedSha) { + phase = "ClosingOut"; + decision = "target sha matches observed source sha but native closeout gates are not complete"; + } else { + phase = "PendingTrigger"; + decision = targetSha === null + ? "target sha is unknown; trigger-current adapter will dedupe by source snapshot" + : `observed ${shortSha(observedSha)} differs from target ${shortSha(targetSha)}`; + } + + if (options.confirm && (phase === "PendingTrigger" || phase === "Superseded" || (phase === "Observed" && observedSha !== null))) { + const trigger = await executeTrigger(registry, follower, observedSha, options); + triggerCommand = trigger.command; + phase = trigger.ok ? (options.wait ? "ClosingOut" : "Triggering") : "Failed"; + decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}: ${redactText(trigger.message).slice(0, 220)}`; + inFlightJob = trigger.jobId ?? live.inFlightJob; + lastTriggeredSha = observedSha; + if (trigger.ok && options.wait && trigger.completed) { + phase = "Succeeded"; + decision = `trigger completed for ${shortSha(observedSha)}`; + inFlightJob = null; + targetSha = observedSha; + lastSucceededSha = observedSha; + } + if (!trigger.ok) warnings.push(trigger.message); + } + if (options.confirm && options.wait && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) { + const closeout = await waitNativeFollowerCloseout(registry, follower, observedSha, options, options.timeoutSeconds ?? follower.budgets.endToEndSeconds); + triggerCommand = closeoutOnlyCommand(follower, live.pipelineRun, observedSha, closeout); + if (closeout.completed) { + phase = "Succeeded"; + decision = `closeout completed for ${shortSha(observedSha)}`; + inFlightJob = null; + targetSha = observedSha; + lastSucceededSha = observedSha; + } else { + decision = closeout.timedOut + ? `closeout did not converge for ${shortSha(observedSha)} within budget` + : `closeout remains pending for ${shortSha(observedSha)}`; + warnings.push(decision); + } + } + + if (options.dryRun && phase === "PendingTrigger") decision = `${decision}; dry-run did not trigger`; + if (shouldRefreshAutomaticCloseout(follower, observedSha, live, phase, options)) { + const refresh = runNativeArgoRefresh(follower.nativeStatus.argo as NonNullable); + if (refresh.exitCode !== 0) warnings.push(`argo refresh failed: ${redactText(tailText(refresh.stderr || refresh.stdout, 300))}`); + } + if (shouldFlushAutomaticCloseout(follower, observedSha, live, phase, options)) { + const gitMirror = asOptionalRecord(asOptionalRecord(live.payload)?.gitMirror); + const flushKey = stringOrNull(gitMirror?.localGitops) ?? observedSha; + const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", follower.budgets.sourceSyncSeconds, flushKey); + if (flush !== null && !flush.result.ok) warnings.push(`git-mirror flush failed: ${redactText(tailText(flush.result.conditionMessage ?? flush.result.logsTail ?? "unknown", 300))}`); + } + const statePipelineRun = stringOrNull(triggerCommand?.pipelineRun) ?? live.pipelineRun; + + return { + id: follower.id, + adapter: follower.adapter, + enabled: follower.enabled, + phase, + source: { + repository: follower.source.repository, + branch: follower.source.branch, + branchRef: follower.source.branchRef, + snapshotPrefix: follower.source.snapshotPrefix, + observedSha, + }, + target: { + node: follower.target.node, + lane: follower.target.lane, + namespace: follower.target.namespace, + sentinel: follower.target.sentinel, + targetSha, + }, + lastTriggeredSha, + lastSucceededSha, + pipelineRun: statePipelineRun, + inFlightJob, + budgetSource: follower.budgets, + controller: { + mode: options.inCluster ? "k8s-controller" : "local-cli", + stateConfigMap: registry.controller.stateConfigMapName, + leaseName: registry.controller.leaseName, + }, + decision, + dryRun: options.dryRun, + updatedAt: new Date().toISOString(), + timings: buildFollowerTimings(follower, live, triggerCommand, asOptionalRecord(previous.timings), phase), + warnings, + next: followerNextCommands(follower), + command: triggerCommand ?? { + status: live.command, + exitCode: live.exitCode, + timedOut: live.timedOut, + payload: live.payload, + }, + }; +} + +function shouldRefreshAutomaticCloseout( + follower: FollowerSpec, + observedSha: string | null, + live: AdapterSummary, + phase: BranchFollowerPhase, + options: ParsedOptions, +): boolean { + if (!options.inCluster || !options.confirm || options.wait || options.dryRun) return false; + if (phase !== "ClosingOut" || observedSha === null || follower.nativeStatus.argo === null) return false; + const payload = asOptionalRecord(live.payload); + const tekton = asOptionalRecord(payload?.tekton); + if (tekton?.succeeded !== true) return false; + const argo = asOptionalRecord(payload?.argo); + const runtime = asOptionalRecord(payload?.runtime); + const argoReady = argo?.ready === true; + const runtimeAligned = runtime?.aligned === true; + return !argoReady || !runtimeAligned || live.targetSha !== observedSha; +} + +function shouldFlushAutomaticCloseout( + follower: FollowerSpec, + observedSha: string | null, + live: AdapterSummary, + phase: BranchFollowerPhase, + options: ParsedOptions, +): boolean { + if (!options.inCluster || !options.confirm || options.wait || options.dryRun) return false; + if (phase !== "ClosingOut" || observedSha === null) return false; + const payload = asOptionalRecord(live.payload); + const gitMirror = asOptionalRecord(payload?.gitMirror); + if (!shouldFlushNativeGitMirrorDuringCloseout(follower, gitMirror)) return false; + const argo = asOptionalRecord(payload?.argo); + const runtime = asOptionalRecord(payload?.runtime); + const argoReady = argo === null || argo.ready === true; + const runtimeAligned = runtime === null ? live.targetSha === observedSha : runtime.aligned === true; + return argoReady && runtimeAligned; +} + +async function executeTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions): Promise { + const spec = follower.commands.trigger; + const timeoutSeconds = options.timeoutSeconds ?? spec.timeoutSeconds; + if (follower.adapter === "hwlab-node-runtime" && options.inCluster) { + return await executeNativeHwlabNodeTrigger(registry, follower, observedSha, options, timeoutSeconds); + } + if (follower.adapter === "agentrun-yaml-lane" && options.inCluster) { + return await executeNativeAgentRunTrigger(registry, follower, observedSha, options, timeoutSeconds); + } + if (follower.adapter === "web-probe-sentinel-cicd" && options.inCluster) { + return await executeNativeSentinelTrigger(registry, follower, observedSha, options, timeoutSeconds); + } + if (!options.wait && !options.inCluster) { + const job = startJob(`cicd_branch_follower_${safeJobSegment(follower.id)}`, spec.argv, `Trigger ${follower.id} for observed sha ${observedSha ?? "unknown"}`); + return { + ok: true, + completed: false, + message: `started async job ${job.id}`, + jobId: job.id, + command: { + mode: "async-job", + argv: spec.argv, + jobId: job.id, + status: `bun scripts/cli.ts job status ${job.id}`, + }, + }; + } + const result = runCommand(spec.argv, repoRoot, { timeoutMs: timeoutSeconds * 1000 }); + return { + ok: result.exitCode === 0, + completed: result.exitCode === 0 && options.wait, + message: result.exitCode === 0 ? "trigger command completed" : tailText(result.stderr || result.stdout, 500), + jobId: null, + command: commandCompact(result, options), + }; +} + +async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions, timeoutSeconds: number): Promise { + if (observedSha === null) { + return nativeTriggerError(follower, "native HWLAB trigger requires observed source sha", "observed-sha-missing"); + } + if (follower.nativeStatus.tekton === null) { + return nativeTriggerError(follower, "native HWLAB trigger requires Tekton nativeStatus", "tekton-native-status-missing"); + } + const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); + const pipelineRun = nodeRuntimePipelineRunName(spec, observedSha); + const namespace = follower.nativeStatus.tekton.namespace; + const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun); + const startedAt = Date.now(); + const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds)); + if (sync !== null && !sync.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt); + } + const refresh = runNativeHwlabControlPlaneRefresh(registry, follower, spec, observedSha, Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.controlPlaneRefreshSeconds), nativeCapabilityJobName(follower.id, "control-plane-refresh", observedSha)); + if (!refresh.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt); + } + const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets); + const payload = parseJsonObject(result.stdout) ?? {}; + const pipelineRunCompleted = payload.completed === true; + const failed = payload.failed === true || result.exitCode !== 0; + const flush = !failed && options.wait && pipelineRunCompleted + ? runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds)) + : null; + if (flush !== null && !flush.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native git-mirror flush failed", startedAt); + } + const closeout = !failed && options.wait && pipelineRunCompleted + ? await waitNativeFollowerCloseout(registry, follower, observedSha, options, remainingSeconds(startedAt, timeoutSeconds)) + : null; + return nativeTektonTriggerResult({ + follower, + observedSha, + namespace, + pipelineRun, + stageRef: `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`, + wait: options.wait, + result, + startedAt, + payload: { + ...payload, + nativeCapabilities: { + gitMirrorSync: sync === null ? null : sync.result, + controlPlaneRefresh: refresh.result, + gitMirrorFlush: flush === null ? null : flush.result, + }, + }, + closeout, + successMessage: `native HWLAB PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}`, + }); +} + +async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions, timeoutSeconds: number): Promise { + if (observedSha === null) { + return nativeTriggerError(follower, "native AgentRun trigger requires observed source sha", "observed-sha-missing"); + } + if (follower.nativeStatus.tekton === null) { + return nativeTriggerError(follower, "native AgentRun trigger requires Tekton nativeStatus", "tekton-native-status-missing"); + } + const { configPath, spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); + const startedAt = Date.now(); + const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`; + const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`; + const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds)); + if (sync !== null && !sync.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt); + } + const buildJob = `${jobPrefix}-build-${observedSha.slice(0, 12)}`.slice(0, 63); + const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), spec.deployment.manager.imageBuild.timeoutSeconds), "buildkit", registry.controller.budgets); + const buildPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(build.logsTail) ?? "" }); + const digest = stringOrNull(buildPayload.digest); + const envIdentity = stringOrNull(buildPayload.envIdentity); + if (!build.ok || digest === null || envIdentity === null) { + return nativeK8sStageFailure(follower, observedSha, "image-build", buildJob, build, buildPayload, "native AgentRun image build failed", startedAt); + } + const image = agentRunImageArtifact(spec, { sourceCommit: observedSha, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" }); + const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: observedSha, image }); + const publishJob = `${jobPrefix}-gitops-${observedSha.slice(0, 12)}`.slice(0, 63); + const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish", registry.controller.budgets); + const publishPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(publish.logsTail) ?? "" }); + if (!publish.ok || publishPayload.ok === false || stringOrNull(publishPayload.gitopsCommit) === null) { + return nativeK8sStageFailure(follower, observedSha, "gitops-publish", publishJob, publish, publishPayload, "native AgentRun GitOps publish failed", startedAt); + } + const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds)); + if (flush !== null && !flush.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native AgentRun git-mirror flush failed", startedAt); + } + const pipelineRun = agentRunPipelineRunName(spec, observedSha); + const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets); + const tektonPayload = parseJsonObject(tektonResult.stdout) ?? {}; + const pipelineRunCompleted = tektonPayload.completed === true; + const failed = tektonPayload.failed === true || tektonResult.exitCode !== 0; + const closeout = !failed && options.wait && pipelineRunCompleted + ? await waitNativeFollowerCloseout(registry, follower, observedSha, options, remainingSeconds(startedAt, timeoutSeconds)) + : null; + const trigger = nativeTektonTriggerResult({ + follower, + observedSha, + namespace: follower.nativeStatus.tekton.namespace, + pipelineRun, + stageRef, + wait: options.wait, + result: tektonResult, + startedAt, + payload: { + ...tektonPayload, + agentrun: { + configPath, + gitMirrorSync: sync === null ? null : { jobName: sync.jobName, payload: sync.result }, + imageBuild: { jobName: buildJob, result: build, payload: buildPayload }, + gitopsPublish: { jobName: publishJob, result: publish, payload: publishPayload }, + gitMirrorFlush: flush === null ? null : { jobName: flush.jobName, payload: flush.result }, + }, + }, + closeout, + successMessage: `native AgentRun PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}`, + }); + return trigger; +} + +function nativeTriggerError(follower: FollowerSpec, message: string, reason: string): TriggerResult { + return { + ok: false, + completed: false, + message, + jobId: null, + command: { mode: "k8s-native-tekton", adapter: follower.adapter, ok: false, reason, parsedDownstreamCliOutput: false }, + }; +} + +function nativeK8sStageFailure( + follower: FollowerSpec, + observedSha: string, + phase: string, + jobName: string, + job: NativeK8sJobResult, + payload: Record, + message: string, + startedAt?: number, +): TriggerResult { + const detail = [ + message, + job.timedOut ? "timedOut=true" : null, + job.conditionReason === null ? null : `reason=${job.conditionReason}`, + job.conditionMessage === null ? null : `condition=${job.conditionMessage}`, + job.logsTail === null ? null : `logs=${tailText(job.logsTail, 500)}`, + ].filter((item): item is string => item !== null).join("; "); + return { + ok: false, + completed: false, + message: detail, + jobId: jobName, + command: { + mode: "k8s-native-job", + adapter: follower.adapter, + phase, + jobName, + sourceCommit: observedSha, + ok: false, + startedAt: startedAt === undefined ? null : new Date(startedAt).toISOString(), + finishedAt: new Date().toISOString(), + elapsedMs: startedAt === undefined ? job.elapsedMs : Date.now() - startedAt, + payload, + job, + statusAuthority: "kubernetes-api-serviceaccount", + parsedDownstreamCliOutput: false, + }, + }; +} + +function runNativeGitMirrorStage(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string, action: "sync" | "flush", timeoutSeconds: number, jobKey = observedSha): { jobName: string; namespace: string; result: NativeK8sJobResult } | null { + const job = nativeGitMirrorJobForFollower(follower, jobKey, action); + if (job === null) return null; + const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action, registry.controller.budgets); + return { jobName: job.jobName, namespace: job.namespace, result }; +} + +function nativeGitMirrorJobForFollower( + follower: FollowerSpec, + observedSha: string, + action: "sync" | "flush", +): { namespace: string; jobName: string; manifest: Record } | null { + const jobName = nativeCapabilityJobName(follower.id, action, observedSha); + if (follower.adapter === "hwlab-node-runtime") { + const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); + const mirror = nodeRuntimeGitMirrorTarget(spec); + return { + namespace: mirror.namespace, + jobName, + manifest: nodeRuntimeGitMirrorJobManifest(mirror, action, jobName), + }; + } + if (follower.adapter === "agentrun-yaml-lane") { + const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); + return { + namespace: spec.gitMirror.namespace, + jobName, + manifest: yamlLaneGitMirrorJobManifest(spec, action, jobName), + }; + } + return null; +} + +function nativeTektonTriggerResult(input: { + follower: FollowerSpec; + observedSha: string; + namespace: string; + pipelineRun: string; + stageRef: string; + wait: boolean; + result: CommandResult; + startedAt: number; + payload: Record; + closeout: NativeCloseoutWaitResult | null; + successMessage: string; +}): TriggerResult { + const pipelineRunCompleted = input.payload.completed === true; + const failed = input.payload.failed === true || input.result.exitCode !== 0; + const stillRunning = input.payload.stillRunning === true || input.payload.timedOutWait === true; + const message = failed + ? nativeTektonFailureText(input.payload, input.result) + : input.closeout?.completed === true + ? input.successMessage + : input.closeout?.timedOut === true + ? `native PipelineRun ${input.pipelineRun} succeeded but runtime closeout did not converge within budget` + : pipelineRunCompleted + ? `native PipelineRun ${input.pipelineRun} succeeded; runtime closeout remains k8s-native` + : stillRunning + ? `native PipelineRun ${input.pipelineRun} is still running; query status/events/logs for closeout` + : `native PipelineRun ${input.pipelineRun} submitted`; + const ok = !failed && (input.closeout === null || input.closeout.completed === true); + const finishedAt = failed || input.result.timedOut || input.closeout?.completed === true || input.closeout?.timedOut === true + ? new Date().toISOString() + : null; + return { + ok, + completed: input.closeout?.completed === true, + message, + jobId: input.pipelineRun, + command: { + mode: "k8s-native-tekton", + adapter: input.follower.adapter, + namespace: input.namespace, + pipelineRun: input.pipelineRun, + sourceCommit: input.observedSha, + sourceStageRef: input.stageRef, + wait: input.wait, + pipelineRunCompleted, + stillRunning, + startedAt: new Date(input.startedAt).toISOString(), + finishedAt, + elapsedMs: Date.now() - input.startedAt, + closeout: input.closeout, + statusAuthority: "kubernetes-api-serviceaccount", + parsedDownstreamCliOutput: false, + payload: input.payload, + exitCode: input.result.exitCode, + timedOut: input.result.timedOut, + stderrTail: failed ? redactText(tailText(input.result.stderr, 1000)) : "", + }, + }; +} + +function nativeTektonFailureText(payload: Record, result: CommandResult): string { + return [ + stringOrNull(payload.message), + stringOrNull(payload.conditionReason), + stringOrNull(payload.conditionMessage), + payload.failed === true ? "PipelineRun failed" : null, + result.timedOut ? "timedOut=true" : null, + ].filter((item): item is string => item !== null && item.length > 0).join("; ") + || "native PipelineRun failed; query events/logs for structured details"; +} + +function remainingSeconds(startedAt: number, timeoutSeconds: number): number { + return Math.max(1, timeoutSeconds - Math.ceil((Date.now() - startedAt) / 1000)); +} + +async function executeNativeSentinelTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions, timeoutSeconds: number): Promise { + if (observedSha === null) { + return { + ok: false, + completed: false, + message: "native sentinel trigger requires observed source sha", + jobId: null, + command: { mode: "k8s-native-tekton", ok: false, reason: "observed-sha-missing", parsedDownstreamCliOutput: false }, + }; + } + if (follower.target.sentinel === null) { + return { + ok: false, + completed: false, + message: "native sentinel trigger requires target.sentinel", + jobId: null, + command: { mode: "k8s-native-tekton", ok: false, reason: "target-sentinel-missing", parsedDownstreamCliOutput: false }, + }; + } + const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); + const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`; + const state = loadSentinelCicdState(spec, follower.target.sentinel, timeoutSeconds, "cached", { + commit: observedSha, + stageRef, + mirrorCommit: observedSha, + sourceAuthority: "git-mirror-snapshot", + }); + const pipelineRun = sentinelPipelineRunName(state, false); + const namespace = stringField(recordField(state.cicd, "builder", `${follower.id}.sentinel.cicd`), "namespace", `${follower.id}.sentinel.cicd.builder`); + const manifest = sentinelPublishPipelineRunManifest(state, pipelineRun, true); + const startedAt = Date.now(); + const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds, registry.controller.budgets); + const payload = parseJsonObject(result.stdout) ?? {}; + const pipelineRunCompleted = payload.completed === true; + const failed = payload.failed === true || result.exitCode !== 0; + const stillRunning = payload.stillRunning === true || payload.timedOutWait === true; + const remainingSeconds = Math.max(1, timeoutSeconds - Math.ceil((Date.now() - startedAt) / 1000)); + const closeout = !failed && options.wait && pipelineRunCompleted + ? await waitNativeSentinelCloseout(registry, follower, observedSha, options, remainingSeconds) + : null; + const message = failed + ? tailText(result.stderr || stringOrNull(payload.message) || result.stdout, 500) + : closeout?.completed === true + ? `native sentinel PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}` + : closeout?.timedOut === true + ? `native sentinel PipelineRun ${pipelineRun} succeeded but runtime closeout did not converge within budget` + : pipelineRunCompleted + ? `native sentinel PipelineRun ${pipelineRun} succeeded; runtime closeout remains k8s-native` + : stillRunning + ? `native sentinel PipelineRun ${pipelineRun} is still running; query status/events/logs for closeout` + : `native sentinel PipelineRun ${pipelineRun} submitted`; + const ok = !failed && (closeout === null || closeout.completed === true); + const finishedAt = failed || result.timedOut || closeout?.completed === true || closeout?.timedOut === true + ? new Date().toISOString() + : null; + return { + ok, + completed: closeout?.completed === true, + message, + jobId: pipelineRun, + command: { + mode: "k8s-native-tekton", + adapter: follower.adapter, + namespace, + pipelineRun, + sourceCommit: observedSha, + sourceStageRef: stageRef, + wait: options.wait, + pipelineRunCompleted, + stillRunning, + startedAt: new Date(startedAt).toISOString(), + finishedAt, + elapsedMs: Date.now() - startedAt, + closeout, + statusAuthority: "kubernetes-api-serviceaccount", + parsedDownstreamCliOutput: false, + payload, + exitCode: result.exitCode, + timedOut: result.timedOut, + stderrTail: failed ? redactText(tailText(result.stderr, Math.min(options.tailBytes, 4000))) : "", + }, + }; +} + +async function waitNativeSentinelCloseout( + registry: BranchFollowerRegistry, + follower: FollowerSpec, + observedSha: string, + options: ParsedOptions, + timeoutSeconds: number, +): Promise { + const refreshResult = follower.nativeStatus.argo === null ? null : runNativeArgoRefresh(follower.nativeStatus.argo); + const startedAt = Date.now(); + const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000; + let polls = 0; + let latest: AdapterSummary | null = null; + let gitMirrorFlush: Record | null = null; + while (Date.now() <= deadline) { + polls += 1; + const remainingSeconds = Math.max(1, Math.ceil((deadline - Date.now()) / 1000)); + latest = await readAdapterStatus(registry, follower, { ...options, timeoutSeconds: Math.min(10, remainingSeconds) }); + const latestPayload = asOptionalRecord(latest.payload); + const latestGitMirror = asOptionalRecord(latestPayload?.gitMirror); + if (latest.observedSha === observedSha && asOptionalRecord(latestPayload?.tekton)?.succeeded === true && gitMirrorFlush === null && shouldFlushNativeGitMirrorDuringCloseout(follower, latestGitMirror)) { + const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", Math.min(remainingSeconds, follower.budgets.sourceSyncSeconds), stringOrNull(latestGitMirror?.localGitops) ?? observedSha); + gitMirrorFlush = flush === null ? null : { + jobName: flush.jobName, + namespace: flush.namespace, + result: compactNativeK8sJobResult(flush.result), + }; + if (flush !== null && !flush.result.ok) { + return { + ok: false, + completed: false, + timedOut: flush.result.timedOut, + polls, + elapsedMs: Date.now() - startedAt, + refresh: refreshResult === null ? null : commandCompact(refreshResult, options), + gitMirrorFlush, + summary: nativeCloseoutSummary(latest), + statusAuthority: "k8s-native", + parsedDownstreamCliOutput: false, + }; + } + continue; + } + if (latest.observedSha === observedSha && latest.aligned === true) { + return { + ok: true, + completed: true, + timedOut: false, + polls, + elapsedMs: Date.now() - startedAt, + refresh: refreshResult === null ? null : commandCompact(refreshResult, options), + gitMirrorFlush, + summary: nativeCloseoutSummary(latest), + statusAuthority: "k8s-native", + parsedDownstreamCliOutput: false, + }; + } + if (Date.now() + 2_000 > deadline) break; + runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); + } + return { + ok: false, + completed: false, + timedOut: true, + polls, + elapsedMs: Date.now() - startedAt, + refresh: refreshResult === null ? null : commandCompact(refreshResult, options), + gitMirrorFlush, + summary: latest === null ? null : nativeCloseoutSummary(latest), + statusAuthority: "k8s-native", + parsedDownstreamCliOutput: false, + }; +} + +function shouldFlushNativeGitMirrorDuringCloseout(follower: FollowerSpec, gitMirror: Record | null): boolean { + if (!nativeGitMirrorRequired(follower) || gitMirror === null) return false; + if (gitMirror.sourceSnapshotReady === false) return false; + return gitMirror.pendingFlush === true || gitMirror.githubInSync === false; +} + +function compactNativeK8sJobResult(result: NativeK8sJobResult): Record { + return { + ok: result.ok, + completed: result.completed, + failed: result.failed, + timedOut: result.timedOut, + created: result.created, + reused: result.reused, + jobName: result.jobName, + namespace: result.namespace, + polls: result.polls, + elapsedMs: result.elapsedMs, + conditionReason: result.conditionReason, + conditionMessage: result.conditionMessage, + statusAuthority: result.statusAuthority, + parsedDownstreamCliOutput: false, + }; +} + +async function waitNativeFollowerCloseout( + registry: BranchFollowerRegistry, + follower: FollowerSpec, + observedSha: string, + options: ParsedOptions, + timeoutSeconds: number, +): Promise { + return await waitNativeSentinelCloseout(registry, follower, observedSha, options, timeoutSeconds); +} + +function closeoutOnlyCommand(follower: FollowerSpec, pipelineRun: string | null, observedSha: string, closeout: NativeCloseoutWaitResult): Record { + return { + mode: "k8s-native-closeout", + adapter: follower.adapter, + pipelineRun, + sourceCommit: observedSha, + wait: true, + closeout, + finishedAt: closeout.completed || closeout.timedOut ? new Date().toISOString() : null, + elapsedMs: closeout.elapsedMs, + exitCode: closeout.completed ? 0 : 1, + timedOut: closeout.timedOut, + statusAuthority: "k8s-native", + parsedDownstreamCliOutput: false, + }; +} + +function nativeCloseoutSummary(live: AdapterSummary): Record { + const payload = asOptionalRecord(live.payload); + return { + ok: live.ok, + phase: live.phase, + observedSha: live.observedSha, + targetSha: live.targetSha, + aligned: live.aligned, + pipelineRun: live.pipelineRun, + pipelineRunPresent: live.pipelineRunPresent, + message: live.message, + gitMirror: asOptionalRecord(payload?.gitMirror), + tekton: asOptionalRecord(payload?.tekton), + taskRuns: compactTaskRunsPayload(asOptionalRecord(payload?.taskRuns)), + planArtifacts: compactPlanArtifactsPayload(asOptionalRecord(payload?.planArtifacts)), + argo: asOptionalRecord(payload?.argo), + runtime: asOptionalRecord(payload?.runtime), + errors: Array.isArray(payload?.errors) ? payload.errors.slice(0, 5) : [], + }; +} + +function runNativeArgoRefresh(argo: NonNullable): CommandResult { + const patchBase64 = Buffer.from(JSON.stringify({ + metadata: { + annotations: { + "argocd.argoproj.io/refresh": "hard", + }, + }, + }), "utf8").toString("base64"); + const script = [ + "set -eu", + `ARGO_NAMESPACE=${shQuote(argo.namespace)}`, + `ARGO_APPLICATION=${shQuote(argo.application)}`, + `PATCH_B64=${shQuote(patchBase64)}`, + "export ARGO_NAMESPACE ARGO_APPLICATION PATCH_B64", + "node <<'NODE_ARGO_REFRESH'", + "import { readFileSync } from 'node:fs';", + "import https from 'node:https';", + "const namespace = process.env.ARGO_NAMESPACE || '';", + "const application = process.env.ARGO_APPLICATION || '';", + "const patch = Buffer.from(process.env.PATCH_B64 || '', 'base64').toString('utf8');", + "const host = process.env.KUBERNETES_SERVICE_HOST;", + "const port = Number(process.env.KUBERNETES_SERVICE_PORT || '443');", + "const token = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/token', 'utf8').trim();", + "const ca = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt');", + "const path = `/apis/argoproj.io/v1alpha1/namespaces/${encodeURIComponent(namespace)}/applications/${encodeURIComponent(application)}`;", + "const req = https.request({ host, port, path, method: 'PATCH', ca, headers: { authorization: `Bearer ${token}`, 'content-type': 'application/merge-patch+json', 'content-length': Buffer.byteLength(patch) } }, (res) => {", + " let body = '';", + " res.setEncoding('utf8');", + " res.on('data', (chunk) => { body += chunk; });", + " res.on('end', () => {", + " if ((res.statusCode || 0) < 200 || (res.statusCode || 0) >= 300) { process.stderr.write(body || `kube api PATCH argo application status ${res.statusCode}`); process.exit(1); }", + " process.stdout.write(JSON.stringify({ ok: true, namespace, application, refresh: 'hard', statusAuthority: 'kubernetes-api-serviceaccount', parsedDownstreamCliOutput: false, valuesRedacted: true }));", + " });", + "});", + "req.on('error', (error) => { process.stderr.write(error?.message || String(error)); process.exit(1); });", + "req.write(patch);", + "req.end();", + "NODE_ARGO_REFRESH", + ].join("\n"); + return runCommand(["sh", "-lc", script], repoRoot, { timeoutMs: 10_000 }); +} + +async function readAdapterStatus(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Promise { + const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.statusSeconds; + const startedAt = Date.now(); + const sourceSync = runNativeSourceObservationSync(registry, follower, options, Math.min(timeoutSeconds, follower.budgets.sourceSyncSeconds)); + const sourceSyncDetail = sourceSync === null || sourceSync.result.ok ? null : redactText(tailText(sourceSync.result.conditionMessage ?? sourceSync.result.logsTail ?? "unknown", 800)); + const sourceSyncError = sourceSyncDetail === null ? null : `native source sync failed: ${sourceSyncDetail}`; + const bundle = readNativeObjectBundle(registry, follower, options, remainingSeconds(startedAt, timeoutSeconds), runKubeScript); + const observedSha = sourceSyncError === null ? stringOrNull(bundle.source?.commit) : null; + const runtimeTargetSha = runtimeTargetShaFromWorkloads(follower.nativeStatus.runtime, bundle.workloads); + const pipelineRunName = stringOrNull(asOptionalRecord(bundle.pipelineRun?.metadata)?.name) ?? expectedPipelineRunName(follower, observedSha); + const pipelineRunPresent = follower.nativeStatus.tekton === null ? null : bundle.pipelineRun !== null; + const pipelineSucceeded = pipelineRunSucceeded(bundle.pipelineRun); + const argoReady = follower.nativeStatus.argo === null ? null : argoApplicationReady(bundle.argoApplication); + const runtimeReady = follower.nativeStatus.runtime === null ? null : runtimeWorkloadsReady(follower.nativeStatus.runtime, bundle.workloads); + const gitMirrorRequired = nativeGitMirrorRequired(follower); + const gitMirrorReady = gitMirrorRequired ? nativeGitMirrorReady(bundle.gitMirror) : null; + const hasTektonGate = follower.nativeStatus.tekton !== null; + const hasRuntimeTarget = runtimeTargetSha !== null; + const requiresRuntimeTarget = follower.nativeStatus.runtime !== null; + const hasTargetEvidence = hasRuntimeTarget || (!requiresRuntimeTarget && hasTektonGate); + const pipelineGateOk = !hasTektonGate + || pipelineSucceeded === true + || (hasRuntimeTarget && runtimeTargetSha === observedSha && argoReady !== false && runtimeReady !== false); + const aligned = observedSha !== null + && hasTargetEvidence + && (requiresRuntimeTarget ? hasRuntimeTarget && runtimeTargetSha === observedSha : true) + && pipelineGateOk + && gitMirrorReady !== false + && argoReady !== false + && runtimeReady !== false; + const targetSha = hasRuntimeTarget && runtimeTargetSha === observedSha && aligned ? runtimeTargetSha : runtimeTargetSha; + const ok = bundle.ok && sourceSyncError === null; + const phase = inferPhase(ok, aligned, observedSha, targetSha, bundle.timedOut); + const errors = sourceSyncError === null ? bundle.errors : [sourceSyncError, ...bundle.errors]; + return { + ok, + command: "native:k8s-git-mirror+tekton+argocd+runtime", + exitCode: bundle.exitCode, + timedOut: bundle.timedOut, + observedSha, + targetSha, + lastTriggeredSha: null, + lastSucceededSha: aligned && observedSha !== null ? observedSha : null, + pipelineRun: pipelineRunName, + pipelineRunPresent, + inFlightJob: pipelineRunPresent === true && pipelineSucceeded === null && pipelineRunName !== null ? pipelineRunName : null, + aligned, + phase, + message: nativeStatusMessage(ok, phase, observedSha, targetSha, { + pipelineSucceeded, + gitMirrorReady, + argoReady, + runtimeReady, + errors, + }), + payload: { + source: bundle.source, + sourceSync: sourceSync === null ? null : sourceSync.result, + gitMirror: nativeGitMirrorSummary(bundle.gitMirror), + tekton: nativePipelineRunSummary(bundle.pipelineRun), + taskRuns: bundle.taskRuns, + planArtifacts: bundle.planArtifacts, + argo: nativeArgoSummary(bundle.argoApplication), + runtime: nativeRuntimeSummary(follower.nativeStatus.runtime, bundle.workloads, observedSha), + timings: { statusRead: { elapsedMs: bundle.elapsedMs, budgetSeconds: timeoutSeconds } }, + errors, + statusAuthority: "k8s-native", + parsedDownstreamCliOutput: false, + }, + stderrTail: bundle.stderrTail, + stdoutTail: bundle.stdoutTail, + }; +} + +function runNativeSourceObservationSync(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, timeoutSeconds: number): { jobName: string; namespace: string; result: NativeK8sJobResult } | null { + if (!options.inCluster) return null; + if (follower.adapter !== "hwlab-node-runtime" && follower.adapter !== "agentrun-yaml-lane") return null; + const bucket = Math.floor(Date.now() / (registry.controller.loop.intervalSeconds * 1000)).toString(36); + return runNativeGitMirrorStage(registry, follower, "source", "sync", timeoutSeconds, `source-${bucket}`); +} + +function inferPhase(ok: boolean, aligned: boolean | null, observedSha: string | null, targetSha: string | null, timedOut: boolean): BranchFollowerPhase { + if (!ok || timedOut) return "Blocked"; + if (aligned === true) return "Succeeded"; + if (observedSha !== null && targetSha !== null && observedSha === targetSha) return "ClosingOut"; + if (observedSha !== null) return "PendingTrigger"; + return "Observed"; +} + +function nativeStatusMessage(ok: boolean, phase: BranchFollowerPhase, observedSha: string | null, targetSha: string | null, gates: { pipelineSucceeded: boolean | null; gitMirrorReady: boolean | null; argoReady: boolean | null; runtimeReady: boolean | null; errors: string[] }): string { + if (!ok) return gates.errors[0] ?? "native status read failed"; + if (phase === "Noop" || phase === "Succeeded") return `target matches ${shortSha(observedSha)}`; + if (observedSha !== null) { + const gatesText = [ + gates.pipelineSucceeded === false ? "pipelineRun not successful" : null, + gates.gitMirrorReady === false ? "git-mirror not flushed" : null, + gates.argoReady === false ? "argo not healthy/synced" : null, + gates.runtimeReady === false ? "runtime not ready" : null, + ].filter((item): item is string => item !== null).join("; "); + if (targetSha !== null) return gatesText.length > 0 ? `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}; ${gatesText}` : `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}`; + return gatesText.length > 0 ? `observed ${shortSha(observedSha)}; ${gatesText}` : `observed ${shortSha(observedSha)} target unknown`; + } + return "k8s-native status did not expose observed source sha"; +} + +function expectedPipelineRunName(follower: FollowerSpec, observedSha: string | null): string | null { + if (observedSha === null || follower.nativeStatus.tekton === null) return null; + return `${follower.nativeStatus.tekton.pipelineRunPrefix}-${shortSha(observedSha)}`; +} + +function mergeFollowerStatus( + registry: BranchFollowerRegistry, + follower: FollowerSpec, + stored: Record, + live: AdapterSummary | null, + liveRequested: boolean, +): Record { + const storedSource = asOptionalRecord(stored.source); + const storedTarget = asOptionalRecord(stored.target); + const phase = live?.phase ?? stringOrNull(stored.phase) ?? "Observed"; + const observedSha = live?.observedSha ?? stringOrNull(storedSource?.observedSha); + const targetSha = live?.targetSha ?? stringOrNull(storedTarget?.targetSha); + const lastTriggeredSha = live?.lastTriggeredSha ?? stringOrNull(stored.lastTriggeredSha); + const lastSucceededSha = live?.lastSucceededSha ?? stringOrNull(stored.lastSucceededSha); + return { + ok: live === null ? true : live.ok, + id: follower.id, + enabled: follower.enabled, + adapter: follower.adapter, + phase, + source: { + repository: follower.source.repository, + branch: follower.source.branch, + observedSha, + snapshotPrefix: follower.source.snapshotPrefix, + }, + target: { + node: follower.target.node, + lane: follower.target.lane, + namespace: follower.target.namespace, + sentinel: follower.target.sentinel, + targetSha, + }, + lastTriggeredSha, + lastSucceededSha, + pipelineRun: live?.pipelineRun ?? stringOrNull(stored.pipelineRun), + inFlightJob: live?.inFlightJob ?? stringOrNull(stored.inFlightJob), + budgetSource: follower.budgets, + updatedAt: stringOrNull(stored.updatedAt), + stateConfigMap: registry.controller.stateConfigMapName, + live: liveRequested, + message: live?.message ?? stringOrNull(stored.decision) ?? "no controller state yet", + timings: live === null ? storedFollowerTimingsForStatus(follower, asOptionalRecord(stored.timings), phase, observedSha) : buildFollowerTimings(follower, live, undefined, asOptionalRecord(stored.timings), phase), + warnings: Array.isArray(stored.warnings) ? stored.warnings.slice(0, 6) : [], + next: followerNextCommands(follower), + }; +} + +function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead { + const errors: string[] = []; + const stateResult = kubeConfigMapFollowerState(registry, options); + const namespace = registry.controller.namespace; + const deploymentResult = kubeJson(registry, options, `kubectl -n ${shQuote(namespace)} get deploy ${shQuote(registry.controller.deploymentName)} -o json`, 10_000, `/apis/apps/v1/namespaces/${encodeURIComponent(namespace)}/deployments/${encodeURIComponent(registry.controller.deploymentName)}`); + const leaseResult = kubeJson(registry, options, `kubectl -n ${shQuote(namespace)} get lease ${shQuote(registry.controller.leaseName)} -o json`, 10_000, `/apis/coordination.k8s.io/v1/namespaces/${encodeURIComponent(namespace)}/leases/${encodeURIComponent(registry.controller.leaseName)}`); + const podSelector = labelSelector(registry.controller.labels); + const podsResult = kubePodList(registry, options, podSelector); + if (!stateResult.ok) errors.push(`state configmap: ${stateResult.error}`); + if (!deploymentResult.ok && !isNotFoundText(deploymentResult.error)) errors.push(`deployment: ${deploymentResult.error}`); + if (!leaseResult.ok && !isNotFoundText(leaseResult.error)) errors.push(`lease: ${leaseResult.error}`); + if (!podsResult.ok && !isNotFoundText(podsResult.error)) errors.push(`pods: ${podsResult.error}`); + return { + ok: errors.length === 0, + stateByFollower: stateResult.stateByFollower, + stateMetadata: stateResult.metadata, + stateValueBytes: stateResult.valueBytes, + stateConfigMapPresent: stateResult.present, + deployment: deploymentResult.value, + lease: leaseResult.value, + pods: podsResult.value, + errors, + }; +} + +function kubeConfigMapFollowerState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sFollowerStateRead { + const followers = options.followerId === null ? registry.followers : registry.followers.filter((follower) => follower.id === options.followerId); + const maxTimingStages = options.followerId === null ? 8 : 16; + const script = [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + nativeCicdScriptLoadShell(["read-state-summary.mjs"]), + `NAMESPACE=${shQuote(registry.controller.namespace)}`, + `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, + `FOLLOWERS_JSON=${shQuote(JSON.stringify(followers.map((follower) => follower.id)))}`, + `MAX_TIMING_STAGES=${maxTimingStages}`, + "export NAMESPACE CONFIGMAP FOLLOWERS_JSON MAX_TIMING_STAGES", + "node \"$tmpdir/read-state-summary.mjs\"", + ].join("\n"); + const result = runKubeScript(registry, options, script, "", 10_000); + const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; + if (parsed === null) { + const error = redactText(tailText(result.stderr || result.stdout, 800)); + return { ok: false, stateByFollower: {}, present: false, error }; + } + const parsedStates = asOptionalRecord(parsed.stateByFollower) ?? {}; + const metadata = asOptionalRecord(parsed.metadata); + const parsedValueBytes = asOptionalRecord(parsed.valueBytes) ?? {}; + const stateByFollower: Record> = {}; + const valueBytes: Record = {}; + for (const follower of followers) { + const state = asOptionalRecord(parsedStates[follower.id]); + if (state !== null) stateByFollower[follower.id] = state; + const bytes = numberOrNull(parsedValueBytes[follower.id]); + if (bytes !== null) valueBytes[follower.id] = bytes; + } + const errors = Array.isArray(parsed.errors) ? parsed.errors.map(String).filter((item) => item.length > 0) : []; + return { + ok: parsed.ok === true && errors.length === 0, + stateByFollower, + metadata, + valueBytes, + present: parsed.present === true, + error: errors.join("; "), + }; +} + +function stateWriteSummary(followerId: string, result: CommandResult): Record { + const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; + return { + follower: followerId, + ok: result.exitCode === 0 && parsed?.ok !== false, + exitCode: result.exitCode, + timedOut: result.timedOut, + beforeResourceVersion: stringOrNull(parsed?.beforeResourceVersion), + afterResourceVersion: stringOrNull(parsed?.afterResourceVersion), + preservedTiming: parsed?.preservedTiming === true, + message: result.exitCode === 0 ? "state patch command completed" : redactText(tailText(result.stderr || result.stdout, 500)), + parsedDownstreamCliOutput: false, + }; +} + +function removeFollowerStateKeys(registry: BranchFollowerRegistry, options: ParsedOptions, ids: string[]): CommandResult { + const patch = JSON.stringify({ data: Object.fromEntries(ids.map((id) => [id, null])) }); + const script = options.inCluster + ? [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + nativeCicdScriptLoadShell(["patch-configmap-data.mjs"]), + `NAMESPACE=${shQuote(registry.controller.namespace)}`, + `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, + `PATCH_B64=${shQuote(Buffer.from(patch, "utf8").toString("base64"))}`, + "export NAMESPACE CONFIGMAP PATCH_B64", + "node \"$tmpdir/patch-configmap-data.mjs\"", + ].join("\n") + : [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + `NAMESPACE=${shQuote(registry.controller.namespace)}`, + `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, + `PATCH=${shQuote(patch)}`, + "export NAMESPACE CONFIGMAP PATCH", + "if ! kubectl -n \"$NAMESPACE\" get configmap \"$CONFIGMAP\" >/dev/null 2>\"$tmpdir/error\"; then", + " if grep -qi 'not found' \"$tmpdir/error\"; then", + " printf '{\"ok\":true,\"present\":false,\"patched\":false,\"reason\":\"state-configmap-not-found\",\"parsedDownstreamCliOutput\":false}'", + " exit 0", + " fi", + " cat \"$tmpdir/error\" >&2", + " exit 1", + "fi", + "kubectl -n \"$NAMESPACE\" patch configmap \"$CONFIGMAP\" --type merge -p \"$PATCH\" >/dev/null", + "printf '{\"ok\":true,\"present\":true,\"patched\":true,\"parsedDownstreamCliOutput\":false}'", + ].join("\n"); + return runKubeScript(registry, options, script, "", 10_000); +} + +function kubeJson(registry: BranchFollowerRegistry, options: ParsedOptions, command: string, timeoutMs: number, inClusterPath?: string): { ok: boolean; value: Record | null; error: string } { + const script = options.inCluster && inClusterPath !== undefined + ? [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + nativeCicdScriptLoadShell(["kube-get.mjs"]), + `node "$tmpdir/kube-get.mjs" ${shQuote(inClusterPath)}`, + ].join("\n") + : `set -eu\n${command}`; + const result = runKubeScript(registry, options, script, "", timeoutMs); + const value = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; + return { + ok: result.exitCode === 0 && value !== null, + value, + error: redactText(tailText(result.stderr || result.stdout, 800)), + }; +} + +function kubePodList(registry: BranchFollowerRegistry, options: ParsedOptions, selector: string): { ok: boolean; value: Record | null; error: string } { + const command = `kubectl -n ${shQuote(registry.controller.namespace)} get pods -l ${shQuote(selector)} -o name`; + const script = options.inCluster + ? [ + "set -eu", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + nativeCicdScriptLoadShell(["kube-get.mjs"]), + `node "$tmpdir/kube-get.mjs" ${shQuote(`/api/v1/namespaces/${encodeURIComponent(registry.controller.namespace)}/pods?labelSelector=${encodeURIComponent(selector)}`)}`, + ].join("\n") + : `set -eu\n${command}`; + const result = runKubeScript(registry, options, script, "", 10_000); + const parsed = options.inCluster && result.exitCode === 0 ? parseJsonObject(result.stdout) : null; + const names = options.inCluster + ? arrayRecords(parsed?.items).map((item) => stringOrNull(asOptionalRecord(item.metadata)?.name)).filter((name): name is string => name !== null) + : result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => line.replace(/^pod\//u, "")); + return { + ok: result.exitCode === 0, + value: result.exitCode === 0 ? { items: names.map((name) => ({ metadata: { name } })) } : null, + error: redactText(tailText(result.stderr || result.stdout, 800)), + }; +} + +function runKubeScript(registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number): CommandResult { + if (options.inCluster) { + return runCommand(["sh", "-lc", script], repoRoot, { input, timeoutMs }); + } + return runCommand([transPath(), registry.controller.kubeRoute, "sh"], repoRoot, { input: `${script}\n`, timeoutMs }); +} + +function compactFollowerStateForConfigMap(state: FollowerState): Record { + return { + id: state.id, + adapter: state.adapter, + enabled: state.enabled, + phase: state.phase, + source: state.source, + target: state.target, + lastTriggeredSha: state.lastTriggeredSha, + lastSucceededSha: state.lastSucceededSha, + pipelineRun: state.pipelineRun, + inFlightJob: state.inFlightJob, + budgetSource: state.budgetSource, + controller: state.controller, + decision: state.decision, + dryRun: state.dryRun, + updatedAt: state.updatedAt, + timings: compactTimings(state.timings), + warnings: compactStateWarnings(state.warnings), + stateFormat: "compact-v1", + command: compactStateCommand(state.command), + }; +} + +function compactStateCommand(command: Record | undefined): Record | undefined { + if (command === undefined) return undefined; + const closeout = asOptionalRecord(command.closeout); + const closeoutSummary = asOptionalRecord(closeout?.summary); + const payload = compactNativePayload(asOptionalRecord(command.payload)) ?? closeoutSummary; + return { + mode: stringOrNull(command.mode) ?? stringOrNull(command.status), + namespace: stringOrNull(command.namespace), + pipelineRun: stringOrNull(command.pipelineRun), + sourceCommit: stringOrNull(command.sourceCommit), + sourceStageRef: stringOrNull(command.sourceStageRef), + wait: command.wait === true ? true : undefined, + pipelineRunCompleted: command.pipelineRunCompleted === true ? true : undefined, + stillRunning: command.stillRunning === true ? true : undefined, + closeout: closeout === null + ? null + : { + ok: closeout.ok === true, + completed: closeout.completed === true, + timedOut: closeout.timedOut === true, + polls: numberOrNull(closeout.polls), + elapsedMs: numberOrNull(closeout.elapsedMs), + gitMirrorFlush: compactCloseoutGitMirrorFlush(asOptionalRecord(closeout.gitMirrorFlush)), + summary: closeoutSummary, + statusAuthority: stringOrNull(closeout.statusAuthority), + parsedDownstreamCliOutput: false, + }, + payload, + exitCode: numberOrNull(command.exitCode), + timedOut: command.timedOut === true, + statusAuthority: stringOrNull(command.statusAuthority), + parsedDownstreamCliOutput: false, + }; +} + +function compactCloseoutGitMirrorFlush(value: Record | null): Record | null { + if (value === null) return null; + const result = asOptionalRecord(value.result); + return { + jobName: stringOrNull(value.jobName), + namespace: stringOrNull(value.namespace), + result: result === null + ? null + : { + ok: result.ok === true, + completed: result.completed === true, + failed: result.failed === true, + timedOut: result.timedOut === true, + created: result.created === true, + reused: result.reused === true, + jobName: stringOrNull(result.jobName), + namespace: stringOrNull(result.namespace), + elapsedMs: numberOrNull(result.elapsedMs), + conditionReason: stringOrNull(result.conditionReason), + conditionMessage: stringOrNull(result.conditionMessage), + statusAuthority: stringOrNull(result.statusAuthority), + parsedDownstreamCliOutput: false, + }, + }; +} + +function compactStateWarnings(warnings: string[]): string[] { + return warnings.slice(0, 4).map((item) => redactText(item.length <= 800 ? item : `${item.slice(0, 400)} ... ${item.slice(-400)}`)); +} + +function compactNativePayload(payload: Record | null): Record | null { + if (payload === null) return null; + return { + source: compactSourcePayload(asOptionalRecord(payload.source)), + sourceSync: compactSourceSyncPayload(asOptionalRecord(payload.sourceSync)), + gitMirror: asOptionalRecord(payload.gitMirror), + tekton: asOptionalRecord(payload.tekton), + taskRuns: compactTaskRunsPayload(asOptionalRecord(payload.taskRuns)), + planArtifacts: compactPlanArtifactsPayload(asOptionalRecord(payload.planArtifacts)), + argo: asOptionalRecord(payload.argo), + runtime: asOptionalRecord(payload.runtime), + errors: Array.isArray(payload.errors) ? payload.errors.slice(0, 5) : [], + statusAuthority: stringOrNull(payload.statusAuthority), + parsedDownstreamCliOutput: false, + }; +} + +function compactSourceSyncPayload(value: Record | null): Record | null { + if (value === null) return null; + return { + ok: value.ok === true, + completed: value.completed === true, + failed: value.failed === true, + timedOut: value.timedOut === true, + created: value.created === true, + reused: value.reused === true, + jobName: stringOrNull(value.jobName), + namespace: stringOrNull(value.namespace), + elapsedMs: numberOrNull(value.elapsedMs), + conditionReason: stringOrNull(value.conditionReason), + conditionMessage: stringOrNull(value.conditionMessage), + statusAuthority: stringOrNull(value.statusAuthority), + parsedDownstreamCliOutput: false, + }; +} + +function compactTaskRunsPayload(taskRuns: Record | null): Record | null { + if (taskRuns === null) return null; + return { + ok: taskRuns.ok === true, + count: numberOrNull(taskRuns.count), + succeededCount: numberOrNull(taskRuns.succeededCount), + failedCount: numberOrNull(taskRuns.failedCount), + activeCount: numberOrNull(taskRuns.activeCount), + performance: asOptionalRecord(taskRuns.performance), + items: Array.isArray(taskRuns.items) ? taskRuns.items.slice(0, 16) : [], + }; +} + +function compactPlanArtifactsPayload(planArtifacts: Record | null): Record | null { + if (planArtifacts === null) return null; + return { + ok: planArtifacts.ok === true, + pipelineRun: stringOrNull(planArtifacts.pipelineRun), + eventFound: planArtifacts.eventFound === true, + degradedReason: stringOrNull(planArtifacts.degradedReason), + sourceCommitId: stringOrNull(planArtifacts.sourceCommitId), + affectedServices: Array.isArray(planArtifacts.affectedServices) ? planArtifacts.affectedServices.slice(0, 40) : [], + rolloutServices: Array.isArray(planArtifacts.rolloutServices) ? planArtifacts.rolloutServices.slice(0, 40) : [], + buildServices: Array.isArray(planArtifacts.buildServices) ? planArtifacts.buildServices.slice(0, 40) : [], + reusedServices: Array.isArray(planArtifacts.reusedServices) ? planArtifacts.reusedServices.slice(0, 40) : [], + buildSkippedCount: numberOrNull(planArtifacts.buildSkippedCount), + summary: stringOrNull(planArtifacts.summary), + disclosure: stringOrNull(planArtifacts.disclosure), + }; +} + +function compactSourcePayload(source: Record | null): Record | null { + if (source === null) return null; + return { + commit: stringOrNull(source.commit), + branch: stringOrNull(source.branch), + repository: stringOrNull(source.repository), + stageRef: stringOrNull(source.stageRef), + sourceAuthority: stringOrNull(source.sourceAuthority), + repoPath: stringOrNull(source.repoPath), + }; +} + +function buildFollowerTimings( + follower: FollowerSpec, + live: AdapterSummary, + triggerCommand: Record | undefined, + storedTimings?: Record | null, + phase?: BranchFollowerPhase, +): FollowerState["timings"] { + const nativePayload = asOptionalRecord(live.payload); + const finishOverride = stringOrNull(triggerCommand?.finishedAt) ?? noopStoredTotalFinishOverride(storedTimings, phase, live); + const total = totalTimingFromCommand(triggerCommand, phase) ?? totalTimingFromStored(storedTimings, phase, finishOverride, live.observedSha); + const stages = dedupeTimingStages([ + ...stageTimingsFromCommand(triggerCommand), + ...stageTimingsFromNativePayload(nativePayload), + ]).slice(0, 24); + const stageSourceCommit = stages.length > 0 ? live.observedSha : null; + return { + budgetSeconds: follower.budgets.endToEndSeconds, + totalSeconds: total?.seconds ?? null, + totalStatus: total?.status ?? "unknown", + totalSource: total?.source ?? "-", + sourceCommit: total?.sourceCommit ?? stringOrNull(triggerCommand?.sourceCommit) ?? stageSourceCommit, + startedAt: total?.startedAt ?? null, + finishedAt: total?.finishedAt ?? null, + overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, + stages, + }; +} + +function noopStoredTotalFinishOverride( + storedTimings: Record | null | undefined, + phase: BranchFollowerPhase | undefined, + live: AdapterSummary, +): string | null { + if (phase !== "Noop" || live.aligned !== true || live.observedSha === null) return null; + if (storedTimings === null || storedTimings === undefined) return null; + if (stringOrNull(storedTimings.sourceCommit) !== live.observedSha) return null; + if (stringOrNull(storedTimings.startedAt) === null) return null; + if (stringOrNull(storedTimings.finishedAt) !== null) return null; + return new Date().toISOString(); +} + +function storedFollowerTimingsForStatus( + follower: FollowerSpec, + storedTimings: Record | null, + phase: BranchFollowerPhase, + observedSha: string | null, +): FollowerState["timings"] { + const total = totalTimingFromStored(storedTimings, phase, null, observedSha); + const sourceCommit = total?.sourceCommit ?? stringOrNull(storedTimings?.sourceCommit) ?? null; + return { + budgetSeconds: follower.budgets.endToEndSeconds, + totalSeconds: total?.seconds ?? null, + totalStatus: total?.status ?? "unknown", + totalSource: total?.source ?? "-", + sourceCommit, + startedAt: total?.startedAt ?? null, + finishedAt: total?.finishedAt ?? null, + overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, + stages: sourceCommit === null ? [] : storedStageTimings(storedTimings), + }; +} + +function storedStageTimings(storedTimings: Record | null): StageTiming[] { + if (storedTimings === null) return []; + return arrayRecords(storedTimings.stages) + .map((stage) => stageTiming( + stringOrNull(stage.stage) ?? "", + stringOrNull(stage.status) ?? "unknown", + numberOrNull(stage.seconds), + numberOrNull(stage.budgetSeconds), + stringOrNull(stage.source) ?? "stored-state", + stringOrNull(stage.object), + )) + .filter((stage) => stage.stage.length > 0) + .slice(0, 24); +} + +function compactTimings(timings: FollowerState["timings"]): FollowerState["timings"] { + return { + budgetSeconds: timings.budgetSeconds, + totalSeconds: timings.totalSeconds, + totalStatus: timings.totalStatus, + totalSource: timings.totalSource, + sourceCommit: timings.sourceCommit, + startedAt: timings.startedAt, + finishedAt: timings.finishedAt, + overBudget: timings.overBudget, + stages: timings.stages.slice(0, 24).map((stage) => ({ + stage: stage.stage, + status: stage.status, + seconds: stage.seconds, + budgetSeconds: stage.budgetSeconds, + source: stage.source, + object: stage.object, + })), + }; +} + +function totalTimingFromCommand(command: Record | undefined, phase?: BranchFollowerPhase): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null { + if (command === undefined) return null; + if (command.mode === "k8s-native-closeout") return null; + const payload = asOptionalRecord(command.payload); + if (payload?.reused === true) return null; + const startedAt = stringOrNull(command.startedAt); + const finishedAt = stringOrNull(command.finishedAt); + const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? secondsFromMs(numberOrNull(command.elapsedMs)); + if (seconds === null) return null; + const closeout = asOptionalRecord(command.closeout); + const exitCode = numberOrNull(command.exitCode); + const status = command.ok === false || (exitCode !== null && exitCode !== 0) + ? "failed" + : command.timedOut === true || closeout?.timedOut === true + ? "timed-out" + : closeout?.completed === true || command.completed === true + ? "completed" + : command.stillRunning === true + ? "running" + : command.pipelineRunCompleted === true + ? "ci-completed" + : phase === undefined + ? "submitted" + : phase.toLowerCase(); + return { seconds, status, source: stringOrNull(command.mode) ?? stringOrNull(command.status) ?? "command", sourceCommit: stringOrNull(command.sourceCommit), startedAt, finishedAt }; +} + +function totalTimingFromStored(storedTimings: Record | null | undefined, phase?: BranchFollowerPhase, finishOverride?: string | null, observedSha?: string | null): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null { + if (storedTimings === null || storedTimings === undefined) return null; + const status = stringOrNull(storedTimings.totalStatus); + const source = stringOrNull(storedTimings.totalSource); + const sourceCommit = stringOrNull(storedTimings.sourceCommit); + if (sourceCommit === null) return null; + if (observedSha !== null && observedSha !== undefined && sourceCommit !== observedSha) return null; + const startedAt = stringOrNull(storedTimings.startedAt); + const finishedAt = stringOrNull(storedTimings.finishedAt) ?? finishOverride ?? null; + if (phase === "Noop" && finishedAt === null) return null; + const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? numberOrNull(storedTimings.totalSeconds); + if (seconds === null) return null; + return { + seconds, + status: finishedAt === null && phase !== undefined && !terminalPhase(phase) ? phase.toLowerCase() : phase === undefined ? status ?? "recorded" : phase.toLowerCase(), + source: source ?? "stored-state", + sourceCommit, + startedAt, + finishedAt, + }; +} + +function totalSecondsFromRange(startedAt: string | null, finishedAt: string | null): number | null { + const startedMs = timestampMs(startedAt); + if (startedMs === null) return null; + const finishedMs = timestampMs(finishedAt) ?? Date.now(); + return finishedMs >= startedMs ? roundSeconds((finishedMs - startedMs) / 1000) : null; +} + +function timestampMs(value: string | null): number | null { + if (value === null) return null; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function terminalPhase(phase: BranchFollowerPhase): boolean { + return phase === "Succeeded" || phase === "Failed" || phase === "Blocked" || phase === "Skipped" || phase === "Noop"; +} + +function stageTimingsFromNativePayload(payload: Record | null): StageTiming[] { + if (payload === null) return []; + const stages: StageTiming[] = []; + const statusRead = asOptionalRecord(asOptionalRecord(payload.timings)?.statusRead); + stages.push(stageTiming("status-read", "ok", secondsFromMs(numberOrNull(statusRead?.elapsedMs)), numberOrNull(statusRead?.budgetSeconds), "native-status", null)); + const sourceSyncStage = k8sJobTiming("git-mirror-sync", asOptionalRecord(payload.sourceSync)); + if (sourceSyncStage !== null) stages.push(sourceSyncStage); + const gitMirror = asOptionalRecord(payload.gitMirror); + if (gitMirror !== null) { + const hasGitopsBranch = stringOrNull(gitMirror.gitopsBranch) !== null; + const sourceReady = gitMirror.sourceSnapshotReady === true; + const status = gitMirror.pendingFlush === true + ? "pending-flush" + : hasGitopsBranch + ? gitMirror.githubInSync === true && sourceReady ? "ready" : "not-ready" + : sourceReady ? "source-ready" : "source-not-ready"; + stages.push(stageTiming("git-mirror", status, null, null, "git-mirror-cache", stringOrNull(gitMirror.gitopsBranch) ?? stringOrNull(gitMirror.sourceBranch))); + } + const tekton = asOptionalRecord(payload.tekton); + if (tekton !== null) { + const status = tekton.succeeded === true ? "succeeded" : tekton.succeeded === false ? `failed:${stringOrNull(tekton.reason) ?? "unknown"}` : "running"; + stages.push(stageTiming("pipelinerun", status, numberOrNull(tekton.durationSeconds), null, "tekton", stringOrNull(tekton.name))); + } + const taskRuns = asOptionalRecord(payload.taskRuns); + const taskRunItems = taskRuns !== null && Array.isArray(taskRuns.items) ? taskRuns.items : []; + for (const item of taskRunItems) { + const record = asOptionalRecord(item); + if (record === null) continue; + const name = stringOrNull(record.pipelineTask) ?? stringOrNull(record.name) ?? "unknown"; + const status = record.status === "True" ? "succeeded" : record.status === "False" ? `failed:${stringOrNull(record.reason) ?? "unknown"}` : "running"; + stages.push(stageTiming(`task:${name}`, status, numberOrNull(record.durationSeconds), null, "tekton-taskrun", stringOrNull(record.name))); + } + const argo = asOptionalRecord(payload.argo); + if (argo !== null) { + stages.push(stageTiming("argo", `${stringOrNull(argo.syncStatus) ?? "unknown"}/${stringOrNull(argo.healthStatus) ?? "unknown"}`, numberOrNull(argo.operationDurationSeconds), null, "argocd", stringOrNull(argo.name))); + } + const runtime = asOptionalRecord(payload.runtime); + if (runtime !== null) { + const aligned = runtime.aligned === true ? "aligned" : runtime.aligned === false ? "stale" : "unknown-target"; + stages.push(stageTiming("runtime", `${runtime.ready === true ? "ready" : "not-ready"}/${aligned}`, null, null, "kubernetes-workload", stringOrNull(runtime.namespace))); + } + return stages; +} + +function stageTimingsFromCommand(command: Record | undefined): StageTiming[] { + if (command === undefined) return []; + const stages: StageTiming[] = []; + const phase = stringOrNull(command.phase); + const jobStage = phase === null ? null : k8sJobTiming(phase, asOptionalRecord(command.job), stringOrNull(command.jobName)); + if (jobStage !== null) stages.push(jobStage); + const payload = asOptionalRecord(command.payload); + if (payload !== null) { + const capabilities = asOptionalRecord(payload.nativeCapabilities); + for (const stage of [ + k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)), + k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)), + k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)), + ]) { + if (stage !== null) stages.push(stage); + } + const agentrun = asOptionalRecord(payload.agentrun); + const agentrunSync = asOptionalRecord(agentrun?.gitMirrorSync); + const agentrunFlush = asOptionalRecord(agentrun?.gitMirrorFlush); + const imageBuild = asOptionalRecord(agentrun?.imageBuild); + const gitopsPublish = asOptionalRecord(agentrun?.gitopsPublish); + for (const stage of [ + k8sJobTiming("git-mirror-sync", asOptionalRecord(agentrunSync?.payload), stringOrNull(agentrunSync?.jobName)), + k8sJobTiming("image-build", asOptionalRecord(imageBuild?.result), stringOrNull(imageBuild?.jobName)), + k8sJobTiming("gitops-publish", asOptionalRecord(gitopsPublish?.result), stringOrNull(gitopsPublish?.jobName)), + k8sJobTiming("git-mirror-flush", asOptionalRecord(agentrunFlush?.payload), stringOrNull(agentrunFlush?.jobName)), + ]) { + if (stage !== null) stages.push(stage); + } + const tektonSeconds = secondsFromMs(numberOrNull(payload.elapsedMs)); + if (tektonSeconds !== null) { + const status = payload.completed === true ? "completed" : payload.failed === true ? "failed" : payload.stillRunning === true ? "running" : "submitted"; + stages.push(stageTiming("pipelinerun-wait", status, tektonSeconds, null, "tekton-submit", stringOrNull(command.pipelineRun))); + } + } + const closeout = asOptionalRecord(command.closeout); + if (closeout !== null) { + const gitMirrorFlush = asOptionalRecord(closeout.gitMirrorFlush); + const gitMirrorFlushStage = k8sJobTiming("git-mirror-flush", asOptionalRecord(gitMirrorFlush?.result), stringOrNull(gitMirrorFlush?.jobName)); + if (gitMirrorFlushStage !== null) stages.push(gitMirrorFlushStage); + const status = closeout.completed === true ? "completed" : closeout.timedOut === true ? "timed-out" : "pending"; + stages.push(stageTiming("closeout", status, secondsFromMs(numberOrNull(closeout.elapsedMs)), null, "k8s-native-closeout", stringOrNull(command.pipelineRun))); + } + return stages; +} + +function k8sJobTiming(stage: string, job: Record | null, objectOverride?: string | null): StageTiming | null { + if (job === null) return null; + const status = job.completed === true + ? job.reused === true ? "reused" : "completed" + : job.failed === true + ? "failed" + : job.timedOut === true + ? "timed-out" + : "running"; + return stageTiming(stage, status, secondsFromMs(numberOrNull(job.elapsedMs)), null, "kubernetes-job", objectOverride ?? stringOrNull(job.jobName)); +} + +function stageTiming(stage: string, status: string, seconds: number | null, budgetSeconds: number | null, source: string, object: string | null): StageTiming { + return { stage, status, seconds, budgetSeconds, source, object }; +} + +function dedupeTimingStages(stages: StageTiming[]): StageTiming[] { + const byKey = new Map(); + for (const stage of stages) { + if (stage.stage.length === 0) continue; + const key = `${stage.stage}\t${stage.object ?? ""}`; + const previous = byKey.get(key); + if (previous === undefined || previous.seconds === null && stage.seconds !== null) byKey.set(key, stage); + } + return [...byKey.values()]; +} + +function secondsFromMs(value: number | null): number | null { + return value === null ? null : roundSeconds(value / 1000); +} + +function roundSeconds(value: number): number { + return Math.round(value * 10) / 10; +} + +function writeFollowerState(registry: BranchFollowerRegistry, state: FollowerState, options: ParsedOptions): CommandResult { + const stateJson = JSON.stringify(compactFollowerStateForConfigMap(state)); + const script = [ + "set -eu", + `NAMESPACE=${shQuote(registry.controller.namespace)}`, + `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, + `FOLLOWER_ID=${shQuote(state.id)}`, + `SPEC_REF=${shQuote(SPEC_REF)}`, + `STATE_B64=${shQuote(Buffer.from(stateJson, "utf8").toString("base64"))}`, + "export NAMESPACE CONFIGMAP FOLLOWER_ID SPEC_REF STATE_B64", + "tmpdir=$(mktemp -d)", + "cleanup() { rm -rf \"$tmpdir\"; }", + "trap cleanup EXIT INT TERM", + nativeCicdScriptLoadShell(["patch-follower-state.mjs"]), + "node \"$tmpdir/patch-follower-state.mjs\"", + ].join("\n"); + return runKubeScript(registry, options, script, "", 10_000); +} + +function runControllerReconcileJob(registry: BranchFollowerRegistry, options: ParsedOptions, mode: { dryRun: boolean; wait: boolean; recordState: boolean }): Record { + const timeoutSeconds = options.timeoutSeconds ?? registry.controller.budgets.runOnceSeconds; + const jobName = `${registry.controller.deploymentName}-once-${Date.now().toString(36)}`.slice(0, 63); + const manifest = renderControllerReconcileJob(registry, options, jobName, mode, timeoutSeconds); + const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`; + const manifestBase64 = Buffer.from(manifestYaml, "utf8").toString("base64"); + const script = [ + "set -eu", + "tmp=$(mktemp)", + "base64 -d >\"$tmp\" <<'UNIDESK_CICD_RECONCILE_JOB_B64'", + manifestBase64, + "UNIDESK_CICD_RECONCILE_JOB_B64", + `kubectl -n ${shQuote(registry.controller.namespace)} delete job ${shQuote(jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`, + `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`, + mode.wait ? waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds) : "true", + ].join("\n"); + const result = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000); + return { + ok: result.exitCode === 0, + name: jobName, + namespace: registry.controller.namespace, + mode: mode.dryRun ? "status-refresh" : "confirm-reconcile", + execution: "k8s-native-job", + exitCode: result.exitCode, + timedOut: result.timedOut, + message: result.exitCode === 0 ? "reconcile job completed" : redactText(tailText(result.stderr || result.stdout, 800)), + stdoutTail: redactText(tailText(result.stdout, options.full ? 4000 : 1000)), + stderrTail: redactText(tailText(result.stderr, options.full ? 2000 : 800)), + }; +} + +function selectFollowers(registry: BranchFollowerRegistry, options: ParsedOptions, opts: { includeDisabled: boolean }): FollowerSpec[] { + let selected = registry.followers; + if (options.followerId !== null) selected = selected.filter((item) => item.id === options.followerId); + else if (!options.all && options.action === "run-once") selected = selected.filter((item) => item.enabled); + if (!opts.includeDisabled) selected = selected.filter((item) => item.enabled); + if (selected.length === 0) throw new Error(options.followerId === null ? "no followers selected" : `unknown or disabled follower ${options.followerId}`); + return selected; +} + +function registrySummary(registry: BranchFollowerRegistry): Record { + return { + path: registry.path, + sha256: registry.rawSha256, + metadata: registry.metadata, + controller: { + namespace: registry.controller.namespace, + kubeRoute: registry.controller.kubeRoute, + deploymentName: registry.controller.deploymentName, + stateConfigMapName: registry.controller.stateConfigMapName, + leaseName: registry.controller.leaseName, + }, + followers: registry.followers.map((item) => item.id), + }; +} + +function redactCommands(follower: FollowerSpec): Record { + return { + plan: follower.commands.plan.argv.join(" "), + status: "native:k8s-git-mirror+tekton+argocd+runtime", + trigger: follower.commands.trigger.argv.join(" "), + events: follower.commands.events.argv.join(" "), + logs: follower.commands.logs.argv.join(" "), + }; +} + +function nativeStatusPlan(native: NativeStatusSpec): Record { + return { + source: { + mode: "k8s-git-mirror-cache", + gitMirrorNamespace: native.source.gitMirrorNamespace, + gitMirrorDeployment: native.source.gitMirrorDeployment, + repoPath: native.source.repoPath, + }, + tekton: native.tekton, + argo: native.argo, + runtime: native.runtime === null + ? null + : { + namespace: native.runtime.namespace, + workloads: native.runtime.workloads.map((item) => ({ kind: item.kind, name: item.name })), + }, + parsedDownstreamCliOutput: false, + }; +} + +function controllerStatusSummary(registry: BranchFollowerRegistry, k8s: K8sStateRead): Record { + const deploymentStatus = asOptionalRecord(k8s.deployment?.status); + const available = numberOrNull(deploymentStatus?.availableReplicas) ?? 0; + const replicas = numberOrNull(deploymentStatus?.replicas) ?? 0; + const leaseSpec = asOptionalRecord(k8s.lease?.spec); + const podItems = Array.isArray(k8s.pods?.items) ? k8s.pods.items.length : null; + return { + namespace: registry.controller.namespace, + route: registry.controller.kubeRoute, + deploymentName: registry.controller.deploymentName, + deploymentPresent: k8s.deployment !== null, + availableReplicas: available, + replicas, + pods: podItems, + stateConfigMapName: registry.controller.stateConfigMapName, + stateConfigMapPresent: k8s.stateConfigMapPresent, + leaseName: registry.controller.leaseName, + leaseHolder: stringOrNull(leaseSpec?.holderIdentity), + noHostWorktreeAuthority: true, + }; +} + +function followerNextCommands(follower: FollowerSpec): Record { + const next: Record = { + status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, + liveStatus: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --live`, + dryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`, + trigger: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --confirm --wait`, + events: `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`, + logs: `bun scripts/cli.ts cicd branch-follower logs --follower ${follower.id}`, + }; + if (follower.nativeStatus.tekton !== null) next.pipelineRuns = `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`; + if (follower.nativeStatus.argo !== null) next.argoApplication = `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`; + return next; +} + +function safeResolveString(ref: string): string | null { + try { + return resolveConfigRefString(ref, ref); + } catch { + return null; + } +} + +function parseJsonObject(text: string): Record | null { + const trimmed = text.trim(); + if (trimmed.length === 0) return null; + try { + const parsed = JSON.parse(trimmed) as unknown; + return asOptionalRecord(parsed); + } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start < 0 || end <= start) return null; + try { + const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown; + return asOptionalRecord(parsed); + } catch { + return null; + } + } +} + +function recordAt(root: Record, path: string[]): Record | null { + let current: unknown = root; + for (const item of path) { + if (typeof current !== "object" || current === null || Array.isArray(current)) return null; + current = (current as Record)[item]; + } + return asOptionalRecord(current); +} + +function asOptionalRecord(value: unknown): Record | null { + if (typeof value !== "object" || value === null || Array.isArray(value)) return null; + return value as Record; +} + +function stringOrNull(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function commandCompact(result: CommandResult, options: ParsedOptions): Record { + return { + argv: result.command, + exitCode: result.exitCode, + timedOut: result.timedOut, + stdoutBytes: Buffer.byteLength(result.stdout), + stderrBytes: Buffer.byteLength(result.stderr), + stdoutTail: options.full || result.exitCode !== 0 ? redactText(tailText(result.stdout, options.tailBytes)) : "", + stderrTail: options.full || result.exitCode !== 0 ? redactText(tailText(result.stderr, Math.min(options.tailBytes, 4000))) : "", + }; +} + +function objectRef(item: Record): Record { + const metadata = asRecord(item.metadata, "metadata"); + return { + kind: stringField(item, "kind", "manifest"), + namespace: typeof metadata.namespace === "string" ? metadata.namespace : "-", + name: stringField(metadata, "name", "manifest.metadata"), + }; +} + +function labelSelector(labels: Record): string { + return Object.entries(labels).map(([key, value]) => `${key}=${value}`).join(","); +} + +function isNotFoundText(value: string): boolean { + return /notfound|not found|notfound|NotFound/u.test(value); +} + +function shortSha(value: string | null): string { + if (value === null) return "-"; + return value.length > 12 ? value.slice(0, 12) : value; +} + +function safeJobSegment(value: string): string { + return value.replace(/[^A-Za-z0-9_.-]/gu, "_").slice(0, 60); +} + +function nativeCapabilityJobName(followerId: string, action: string, sha: string): string { + const prefix = `${safeK8sNameSegment(followerId)}-${safeK8sNameSegment(action)}`; + return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63); +} + +function safeK8sNameSegment(value: string): string { + const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, ""); + return (normalized.length === 0 ? "x" : normalized).slice(0, 40).replace(/-$/u, ""); +} + +function tailText(text: string, maxChars: number): string { + if (text.length <= maxChars) return text; + return text.slice(text.length - maxChars); +} + +function commandLabel(options: ParsedOptions): string { + return `cicd branch-follower ${options.action}`; +} + +function arrayRecords(value: unknown): Record[] { + return Array.isArray(value) ? value.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; +} diff --git a/scripts/src/cicd-native-summary.ts b/scripts/src/cicd-native-summary.ts new file mode 100644 index 00000000..4e37b0c4 --- /dev/null +++ b/scripts/src/cicd-native-summary.ts @@ -0,0 +1,230 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower. +// Responsibility: compact native Kubernetes status summaries for branch-follower closeout. +import { resolveAgentRunLaneTarget } from "./agentrun-lanes"; +import type { FollowerSpec, NativeStatusSpec, NativeWorkloadSpec } from "./cicd-types"; +import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; + +export function pipelineRunSucceeded(pipelineRun: Record | null): boolean | null { + if (pipelineRun === null) return null; + const conditions = Array.isArray(asOptionalRecord(pipelineRun.status)?.conditions) ? asOptionalRecord(pipelineRun.status)?.conditions : []; + for (const condition of conditions as unknown[]) { + const record = asOptionalRecord(condition); + if (record?.type !== "Succeeded") continue; + if (record.status === "True") return true; + if (record.status === "False") return false; + return null; + } + return null; +} + +export function argoApplicationReady(application: Record | null): boolean { + if (application === null) return false; + const status = asOptionalRecord(application.status); + const sync = asOptionalRecord(status?.sync); + const health = asOptionalRecord(status?.health); + return sync?.status === "Synced" && health?.status === "Healthy"; +} + +export function runtimeWorkloadsReady(runtime: NativeStatusSpec["runtime"], workloads: Record[]): boolean { + if (runtime === null) return true; + if (workloads.length < runtime.workloads.length) return false; + return runtime.workloads.every((spec, index) => workloadReady(spec, workloads[index] ?? null)); +} + +export function runtimeTargetShaFromWorkloads(runtime: NativeStatusSpec["runtime"], workloads: Record[]): string | null { + if (runtime === null) return null; + const commits: string[] = []; + runtime.workloads.forEach((spec, index) => { + const commit = workloadSourceCommit(spec, workloads[index] ?? null); + if (commit !== null) commits.push(commit); + }); + const unique = Array.from(new Set(commits)); + return unique.length === 1 ? unique[0] ?? null : null; +} + +export function nativePipelineRunSummary(pipelineRun: Record | null): Record | null { + if (pipelineRun === null) return null; + const metadata = asOptionalRecord(pipelineRun.metadata); + const status = asOptionalRecord(pipelineRun.status); + const condition = latestCondition(status, "Succeeded"); + return { + name: stringOrNull(metadata?.name), + namespace: stringOrNull(metadata?.namespace), + succeeded: pipelineRunSucceeded(pipelineRun), + reason: stringOrNull(condition?.reason), + startTime: stringOrNull(status?.startTime), + completionTime: stringOrNull(status?.completionTime), + durationSeconds: numberOrNull(status?.durationSeconds), + }; +} + +export function nativeArgoSummary(application: Record | null): Record | null { + if (application === null) return null; + const metadata = asOptionalRecord(application.metadata); + const status = asOptionalRecord(application.status); + const sync = asOptionalRecord(status?.sync); + const health = asOptionalRecord(status?.health); + const operationState = asOptionalRecord(status?.operationState); + return { + name: stringOrNull(metadata?.name), + namespace: stringOrNull(metadata?.namespace), + syncStatus: stringOrNull(sync?.status), + healthStatus: stringOrNull(health?.status), + healthMessage: stringOrNull(health?.message), + revision: stringOrNull(sync?.revision), + operationPhase: stringOrNull(operationState?.phase), + operationMessage: stringOrNull(operationState?.message), + operationStartedAt: stringOrNull(operationState?.startedAt), + operationFinishedAt: stringOrNull(operationState?.finishedAt), + operationDurationSeconds: numberOrNull(operationState?.durationSeconds), + conditions: Array.isArray(status?.conditions) ? status.conditions.slice(0, 5) : [], + nonReadyResources: Array.isArray(status?.nonReadyResources) ? status.nonReadyResources.slice(0, 5) : [], + ready: argoApplicationReady(application), + }; +} + +export function nativeRuntimeSummary(runtime: NativeStatusSpec["runtime"], workloads: Record[], expectedSha: string | null): Record | null { + if (runtime === null) return null; + const targetSha = runtimeTargetShaFromWorkloads(runtime, workloads); + return { + namespace: runtime.namespace, + ready: runtimeWorkloadsReady(runtime, workloads), + targetSha, + expectedSha, + aligned: expectedSha !== null && targetSha !== null ? targetSha === expectedSha : null, + workloads: runtime.workloads.map((spec, index) => { + const workload = workloads[index] ?? null; + const sourceCommit = workloadSourceCommit(spec, workload); + return { + kind: spec.kind, + name: spec.name, + ready: workloadReady(spec, workload), + sourceCommit, + aligned: expectedSha !== null && sourceCommit !== null ? sourceCommit === expectedSha : null, + }; + }), + }; +} + +export function nativeGitMirrorRequired(follower: FollowerSpec): boolean { + return nativeGitMirrorGitopsBranch(follower) !== null + && (follower.closeoutChecks.includes("gitMirrorPostFlush") || follower.closeoutChecks.includes("gitops")); +} + +export function nativeGitMirrorReady(gitMirror: Record | null): boolean { + if (gitMirror === null) return false; + if (gitMirror.ok === false) return false; + if (gitMirror.sourceSnapshotReady === false) return false; + if (gitMirror.pendingFlush === true) return false; + if (gitMirror.githubInSync === false) return false; + return true; +} + +export function nativeGitMirrorSummary(gitMirror: Record | null): Record | null { + if (gitMirror === null) return null; + return { + ok: gitMirror.ok === true, + repository: stringOrNull(gitMirror.repository), + sourceBranch: stringOrNull(gitMirror.sourceBranch), + gitopsBranch: stringOrNull(gitMirror.gitopsBranch), + localSource: stringOrNull(gitMirror.localSource), + githubSource: stringOrNull(gitMirror.githubSource), + sourceStageRef: stringOrNull(gitMirror.sourceStageRef), + sourceSnapshotReady: gitMirror.sourceSnapshotReady === true, + localGitops: stringOrNull(gitMirror.localGitops), + githubGitops: stringOrNull(gitMirror.githubGitops), + pendingFlush: gitMirror.pendingFlush === true, + githubInSync: gitMirror.githubInSync === true, + statusAuthority: stringOrNull(gitMirror.statusAuthority) ?? "k8s-git-mirror-cache", + }; +} + +function nativeGitMirrorGitopsBranch(follower: FollowerSpec): string | null { + if (follower.adapter === "hwlab-node-runtime") { + return hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node).gitopsBranch; + } + if (follower.adapter === "agentrun-yaml-lane") { + return resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }).spec.gitops.branch; + } + return null; +} + +function workloadReady(spec: NativeWorkloadSpec, workload: Record | null): boolean { + if (workload === null) return false; + const status = asOptionalRecord(workload.status); + if (spec.kind === "Deployment") { + const desired = numberOrNull(asOptionalRecord(workload.spec)?.replicas) ?? 1; + const available = numberOrNull(status?.availableReplicas) ?? 0; + const updated = numberOrNull(status?.updatedReplicas) ?? 0; + return available >= desired && updated >= desired; + } + const desired = numberOrNull(asOptionalRecord(workload.spec)?.replicas) ?? 1; + const ready = numberOrNull(status?.readyReplicas) ?? 0; + return ready >= desired; +} + +function workloadSourceCommit(spec: NativeWorkloadSpec, workload: Record | null): string | null { + if (workload === null) return null; + const metadata = asOptionalRecord(workload.metadata); + const labels = asOptionalRecord(metadata?.labels); + const annotations = asOptionalRecord(metadata?.annotations); + const template = asOptionalRecord(asOptionalRecord(workload.spec)?.template); + const podMetadata = asOptionalRecord(template?.metadata); + const podLabels = asOptionalRecord(podMetadata?.labels); + const podAnnotations = asOptionalRecord(podMetadata?.annotations); + for (const key of spec.sourceCommit.labels) { + const value = shaOrNull(labels?.[key]); + if (value !== null) return value; + } + for (const key of spec.sourceCommit.annotations) { + const value = shaOrNull(annotations?.[key]); + if (value !== null) return value; + } + for (const key of spec.sourceCommit.podLabels) { + const value = shaOrNull(podLabels?.[key]); + if (value !== null) return value; + } + for (const key of spec.sourceCommit.podAnnotations) { + const value = shaOrNull(podAnnotations?.[key]); + if (value !== null) return value; + } + const containers = Array.isArray(asOptionalRecord(template?.spec)?.containers) ? asOptionalRecord(template?.spec)?.containers : []; + for (const envName of spec.sourceCommit.env) { + for (const container of containers as unknown[]) { + const env = Array.isArray(asOptionalRecord(container)?.env) ? asOptionalRecord(container)?.env : []; + for (const entry of env as unknown[]) { + const record = asOptionalRecord(entry); + if (record?.name !== envName) continue; + const value = shaOrNull(record.value); + if (value !== null) return value; + } + } + } + return null; +} + +function latestCondition(status: Record | null, type: string): Record | null { + const conditions = Array.isArray(status?.conditions) ? status.conditions : []; + for (const condition of conditions as unknown[]) { + const record = asOptionalRecord(condition); + if (record?.type === type) return record; + } + return null; +} + +function shaOrNull(value: unknown): string | null { + return typeof value === "string" && /^[0-9a-f]{40}$/iu.test(value) ? value : null; +} + +function asOptionalRecord(value: unknown): Record | null { + if (typeof value !== "object" || value === null || Array.isArray(value)) return null; + return value as Record; +} + +function stringOrNull(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} diff --git a/scripts/src/cicd.ts b/scripts/src/cicd.ts index 5823fd87..af033850 100644 --- a/scripts/src/cicd.ts +++ b/scripts/src/cicd.ts @@ -1,2769 +1,13 @@ // SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower. -// Responsibility: YAML-first K8s branch-follower controller, status, and adapter orchestration. -import { createHash } from "node:crypto"; -import { readFileSync } from "node:fs"; -import { isAbsolute } from "node:path"; -import { repoRoot, rootPath, type UniDeskConfig } from "./config"; -import { runCommand, type CommandResult } from "./command"; -import { startJob } from "./jobs"; +// Responsibility: thin CI/CD top-level route entry; branch-follower logic lives in responsibility modules. +import type { UniDeskConfig } from "./config"; import type { RenderedCliResult } from "./output"; -import { renderMachine, renderResult } from "./cicd-render"; -import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; -import { agentRunImageArtifact, renderAgentRunGitopsFiles } from "./agentrun-manifests"; -import { agentRunPipelineRunName, resolveAgentRunLaneTarget } from "./agentrun-lanes"; -import { yamlLaneGitMirrorJobManifest, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLanePipelineRunManifest } from "./agentrun/secrets"; -import { yamlLaneK3sBuildImageJobManifest } from "./agentrun/yaml-lane"; -import { nodeRuntimePipelineRunName } from "./hwlab-node/cleanup"; -import { nodeRuntimeGitMirrorJobManifest } from "./hwlab-node/render"; -import { nodeRuntimeGitMirrorTarget, nodeRuntimePipelineRunManifest } from "./hwlab-node/web-probe"; -import { loadSentinelCicdState } from "./hwlab-node-web-sentinel-cicd"; -import { sentinelPublishPipelineRunManifest } from "./hwlab-node-web-sentinel-cicd-jobs"; -import { sentinelPipelineRunName } from "./hwlab-node-web-sentinel-cicd-shared"; -import { transPath } from "./hwlab-node/runtime-common"; -import { configRefGraph, resolveConfigRefString } from "./ops/config-refs"; -import { renderControllerManifests, renderControllerReconcileJob, waitForJobShell } from "./cicd-controller-render"; -import { buildDebugStep } from "./cicd-debug"; -import { runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh"; -import { nativeCicdScriptLoadShell, readNativeObjectBundle } from "./cicd-native-bundle"; -import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native"; -import type { AdapterSummary, BranchFollowerAction, BranchFollowerDebugStep, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types"; -import { - arrayField, - asRecord, - booleanField, - integerField, - readYamlRecord, - recordField, - redactText, - shQuote, - stringArrayField, - stringField, -} from "./platform-infra-ops-library"; - -const DEFAULT_CONFIG_PATH = "config/cicd-branch-followers.yaml"; -const SPEC_REF = "PJ2026-01060703"; -const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower"; +import { cicdHelp as branchFollowerHelp, runCicdCommand as runBranchFollowerCommand } from "./cicd-branch-follower"; export function cicdHelp(): unknown { - return { - command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs", - output: "text by default; use --json, --raw, or -o json|yaml for machine output", - usage: [ - "bun scripts/cli.ts cicd branch-follower plan", - "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", - "bun scripts/cli.ts cicd branch-follower status", - "bun scripts/cli.ts cicd branch-follower status --live", - "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", - "bun scripts/cli.ts cicd branch-follower run-once --follower hwlab-jd01-v03 --confirm --wait", - "bun scripts/cli.ts cicd branch-follower debug-step --follower web-probe-sentinel-master --step state-read", - "bun scripts/cli.ts cicd branch-follower debug-step --follower web-probe-sentinel-master --step state-write --confirm", - "bun scripts/cli.ts cicd branch-follower cleanup-state --follower web-probe-sentinel-master --confirm", - "bun scripts/cli.ts cicd branch-follower events --follower agentrun-jd01-v02", - "bun scripts/cli.ts cicd branch-follower logs --follower web-probe-sentinel-master", - ], - config: DEFAULT_CONFIG_PATH, - spec: `${SPEC_REF} ${SPEC_VERSION}`, - description: "Deploy and inspect the YAML-first Kubernetes branch follower that follows HWLAB v0.3, AgentRun v0.2, and the selected web-probe sentinel master lane without using host worktrees as source authority.", - }; + return branchFollowerHelp(); } -export async function runCicdCommand(_config: UniDeskConfig | null, args: string[]): Promise { - const top = args[0]; - if (top === undefined || isHelpToken(top)) return renderMachine("cicd", cicdHelp(), "json"); - if (top !== "branch-follower") { - throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs"); - } - const options = parseOptions(args.slice(1)); - const command = commandLabel(options); - if (options.action === "help") return renderMachine(command, cicdHelp(), "json"); - const registry = readRegistry(options.configPath); - switch (options.action) { - case "plan": - return renderResult(command, buildPlan(registry, options), options); - case "apply": - return renderResult(command, await applyController(registry, options), options); - case "status": - return renderResult(command, await buildStatus(registry, options), options); - case "run-once": - return renderResult(command, await runOnce(registry, options), options); - case "debug-step": - return renderResult(command, await buildDebugStep(registry, options, { selectFollowers, readK8sState, readAdapterStatus, decideAndMaybeTrigger, writeFollowerState, runKubeScript }), options); - case "cleanup-state": - return renderResult(command, cleanupState(registry, options), options); - case "events": - case "logs": - return renderResult(command, await runFollowerDrillDown(registry, options), options); - case "help": - return renderMachine(command, cicdHelp(), "json"); - } -} - -function parseOptions(args: string[]): ParsedOptions { - const actionToken = args[0]; - if (actionToken === undefined || isHelpToken(actionToken)) { - return defaultOptions("help", args.slice(actionToken === undefined ? 0 : 1)); - } - if (!["plan", "apply", "status", "run-once", "debug-step", "cleanup-state", "events", "logs"].includes(actionToken)) { - throw new Error(`cicd branch-follower unknown action: ${actionToken}`); - } - const action = actionToken as BranchFollowerAction; - const options = defaultOptions(action, []); - const rest = args.slice(1); - for (let index = 0; index < rest.length; index += 1) { - const arg = rest[index] ?? ""; - if (isHelpToken(arg)) { - options.action = "help"; - } else if (arg === "--config") { - options.configPath = valueOption(rest, ++index, arg); - } else if (arg === "--follower" || arg === "--target") { - options.followerId = simpleId(valueOption(rest, ++index, arg), arg); - } else if (arg === "--all") { - options.all = true; - } else if (arg === "--confirm") { - options.confirm = true; - } else if (arg === "--dry-run") { - options.dryRun = true; - } else if (arg === "--wait") { - options.wait = true; - } else if (arg === "--in-cluster") { - options.inCluster = true; - } else if (arg === "--controller") { - if (isInClusterRuntime()) options.inCluster = true; - else if (options.action === "status") options.live = true; - } else if (arg === "--live") { - options.live = true; - } else if (arg === "--no-live") { - options.noLive = true; - } else if (arg === "--full") { - options.full = true; - } else if (arg === "--raw" || arg === "--json") { - options.raw = true; - options.full = true; - options.output = "json"; - } else if (arg === "--record-state") { - options.recordState = true; - } else if (arg === "--step") { - options.debugStep = debugStepOption(valueOption(rest, ++index, arg)); - } else if (arg === "-o" || arg === "--output") { - const value = valueOption(rest, ++index, arg); - if (value !== "json" && value !== "yaml" && value !== "wide" && value !== "text") throw new Error(`${arg} must be json, yaml, wide, or text`); - options.output = value === "wide" || value === "text" ? "human" : value; - if (value === "json" || value === "yaml") { - options.raw = true; - options.full = true; - } - } else if (arg === "--limit") { - options.limit = positiveInt(valueOption(rest, ++index, arg), arg); - } else if (arg === "--tail-bytes" || arg === "--tail") { - options.tailBytes = positiveInt(valueOption(rest, ++index, arg), arg); - } else if (arg === "--timeout-seconds") { - options.timeoutSeconds = positiveInt(valueOption(rest, ++index, arg), arg); - } else { - throw new Error(`unsupported cicd branch-follower option: ${arg}`); - } - } - if (options.confirm && options.dryRun) throw new Error("cicd branch-follower accepts only one of --confirm or --dry-run"); - if (options.action === "apply" && !options.confirm) options.dryRun = true; - if (options.action === "run-once" && !options.confirm) options.dryRun = true; - if (options.action === "debug-step" && !options.confirm) options.dryRun = true; - if (options.action === "cleanup-state" && !options.confirm) options.dryRun = true; - if (options.action === "run-once" && options.confirm && !options.all && options.followerId === null) { - throw new Error("run-once --confirm requires --all or --follower "); - } - if (options.action === "cleanup-state" && options.confirm && !options.all && options.followerId === null) { - throw new Error("cleanup-state --confirm requires --all or --follower "); - } - if (options.action === "debug-step" && options.followerId === null) { - throw new Error("debug-step requires --follower "); - } - return options; -} - -function debugStepOption(value: string): BranchFollowerDebugStep { - if (value === "state-read" || value === "status-read" || value === "decide" || value === "state-write") return value; - throw new Error("--step must be state-read, status-read, decide, or state-write"); -} - -function isInClusterRuntime(): boolean { - return Boolean(process.env.KUBERNETES_SERVICE_HOST && process.env.KUBERNETES_SERVICE_PORT); -} - -function defaultOptions(action: BranchFollowerAction, _args: string[]): ParsedOptions { - return { - action, - configPath: DEFAULT_CONFIG_PATH, - followerId: null, - all: false, - confirm: false, - dryRun: false, - wait: false, - inCluster: false, - live: false, - noLive: false, - full: false, - raw: false, - recordState: false, - debugStep: null, - output: "human", - limit: 20, - tailBytes: 12000, - timeoutSeconds: null, - }; -} - -function valueOption(args: string[], index: number, option: string): string { - const value = args[index]; - if (value === undefined || value.length === 0 || value.startsWith("--")) throw new Error(`${option} requires a value`); - return value; -} - -function simpleId(value: string, option: string): string { - if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${option} must be a simple id`); - return value; -} - -function positiveInt(value: string, option: string): number { - const parsed = Number(value); - if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${option} must be a positive integer`); - return parsed; -} - -function isHelpToken(value: string): boolean { - return value === "-h" || value === "--help" || value === "help"; -} - -function readRegistry(configPath: string): BranchFollowerRegistry { - const absolute = isAbsolute(configPath) ? configPath : rootPath(configPath); - const rawText = readFileSync(absolute, "utf8"); - const root = readYamlRecord>(absolute, "CicdBranchFollowerRegistry"); - const metadata = recordField(root, "metadata", configPath); - const controller = parseController(recordField(root, "controller", configPath)); - const followers = arrayField(root, "followers", configPath).map(parseFollower); - const ids = new Set(); - for (const follower of followers) { - if (ids.has(follower.id)) throw new Error(`${configPath}.followers has duplicate id ${follower.id}`); - ids.add(follower.id); - } - return { - path: configPath, - rawText, - rawSha256: createHash("sha256").update(rawText).digest("hex"), - metadata: { - id: stringField(metadata, "id", `${configPath}.metadata`), - owner: stringField(metadata, "owner", `${configPath}.metadata`), - specRef: stringField(metadata, "specRef", `${configPath}.metadata`), - version: stringField(metadata, "version", `${configPath}.metadata`), - }, - controller, - followers, - }; -} - -function parseController(root: Record): ControllerSpec { - const source = recordField(root, "source", "controller"); - const authority = recordField(source, "sourceAuthority", "controller.source"); - const snapshot = recordField(source, "sourceSnapshot", "controller.source"); - const loop = recordField(root, "loop", "controller"); - const budgets = recordField(root, "budgets", "controller"); - const result: ControllerSpec = { - namespace: stringField(root, "namespace", "controller"), - kubeRoute: stringField(root, "kubeRoute", "controller"), - fieldManager: stringField(root, "fieldManager", "controller"), - serviceAccountName: stringField(root, "serviceAccountName", "controller"), - deploymentName: stringField(root, "deploymentName", "controller"), - configMapName: stringField(root, "configMapName", "controller"), - stateConfigMapName: stringField(root, "stateConfigMapName", "controller"), - leaseName: stringField(root, "leaseName", "controller"), - image: stringField(root, "image", "controller"), - labels: stringMap(recordField(root, "labels", "controller"), "controller.labels"), - source: { - repository: stringField(source, "repository", "controller.source"), - branch: stringField(source, "branch", "controller.source"), - gitMirrorReadUrl: stringField(source, "gitMirrorReadUrl", "controller.source"), - gitMirrorCachePvcName: stringField(source, "gitMirrorCachePvcName", "controller.source"), - githubSsh: parseControllerGithubSsh(recordField(source, "githubSsh", "controller.source")), - sourceAuthority: { - mode: stringField(authority, "mode", "controller.source.sourceAuthority"), - resolver: stringField(authority, "resolver", "controller.source.sourceAuthority"), - allowHostGit: booleanField(authority, "allowHostGit", "controller.source.sourceAuthority"), - allowHostWorkspace: booleanField(authority, "allowHostWorkspace", "controller.source.sourceAuthority"), - allowGithubDirectInPipeline: booleanField(authority, "allowGithubDirectInPipeline", "controller.source.sourceAuthority"), - }, - sourceSnapshot: { - stageRefPrefix: stringField(snapshot, "stageRefPrefix", "controller.source.sourceSnapshot"), - missingObjectPolicy: stringField(snapshot, "missingObjectPolicy", "controller.source.sourceSnapshot"), - refreshPolicy: stringField(snapshot, "refreshPolicy", "controller.source.sourceSnapshot"), - }, - }, - loop: { - intervalSeconds: integerField(loop, "intervalSeconds", "controller.loop"), - reconcileTimeoutSeconds: integerField(loop, "reconcileTimeoutSeconds", "controller.loop"), - leaseDurationSeconds: integerField(loop, "leaseDurationSeconds", "controller.loop"), - terminationGracePeriodSeconds: integerField(loop, "terminationGracePeriodSeconds", "controller.loop"), - }, - budgets: { - applyWaitSeconds: integerField(budgets, "applyWaitSeconds", "controller.budgets"), - statusSeconds: integerField(budgets, "statusSeconds", "controller.budgets"), - runOnceSeconds: integerField(budgets, "runOnceSeconds", "controller.budgets"), - reconcileJobTtlSeconds: integerField(budgets, "reconcileJobTtlSeconds", "controller.budgets"), - reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"), - reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"), - reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"), - nativeTransportGraceSeconds: integerField(budgets, "nativeTransportGraceSeconds", "controller.budgets"), - nativePollIntervalSeconds: integerField(budgets, "nativePollIntervalSeconds", "controller.budgets"), - }, - }; - if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) { - throw new Error("controller.source.sourceAuthority must disable host git, host workspace, and direct GitHub pipeline fallback"); - } - return result; -} - -function parseControllerGithubSsh(root: Record): ControllerSpec["source"]["githubSsh"] { - return { - secretName: stringField(root, "secretName", "controller.source.githubSsh"), - privateKeySecretKey: stringField(root, "privateKeySecretKey", "controller.source.githubSsh"), - proxyHost: stringField(root, "proxyHost", "controller.source.githubSsh"), - proxyPort: integerField(root, "proxyPort", "controller.source.githubSsh"), - }; -} - -function parseFollower(root: Record, index: number): FollowerSpec { - const label = `followers[${index}]`; - const source = recordField(root, "source", label); - const target = recordField(root, "target", label); - const budgets = recordField(root, "budgets", label); - const commands = recordField(root, "commands", label); - const nativeStatus = recordField(root, "nativeStatus", label); - const closeout = recordField(root, "closeout", label); - const configRefs = stringMap(recordField(target, "configRefs", `${label}.target`), `${label}.target.configRefs`); - return { - id: simpleId(stringField(root, "id", label), `${label}.id`), - enabled: booleanField(root, "enabled", label), - adapter: stringField(root, "adapter", label), - description: stringField(root, "description", label), - source: { - repository: stringField(source, "repository", `${label}.source`), - branch: stringField(source, "branch", `${label}.source`), - branchRef: stringField(source, "branchRef", `${label}.source`), - authorityRef: stringField(source, "authorityRef", `${label}.source`), - snapshotPrefix: stringField(source, "snapshotPrefix", `${label}.source`), - snapshotRef: stringField(source, "snapshotRef", `${label}.source`), - }, - target: { - node: stringField(target, "node", `${label}.target`), - lane: stringField(target, "lane", `${label}.target`), - namespace: stringField(target, "namespace", `${label}.target`), - sentinel: typeof target.sentinel === "string" && target.sentinel.length > 0 ? target.sentinel : null, - configRefs, - }, - budgets: { - endToEndSeconds: integerField(budgets, "endToEndSeconds", `${label}.budgets`), - statusSeconds: integerField(budgets, "statusSeconds", `${label}.budgets`), - triggerSeconds: integerField(budgets, "triggerSeconds", `${label}.budgets`), - sourceSyncSeconds: integerField(budgets, "sourceSyncSeconds", `${label}.budgets`), - controlPlaneRefreshSeconds: integerField(budgets, "controlPlaneRefreshSeconds", `${label}.budgets`), - capabilityJobTtlSeconds: integerField(budgets, "capabilityJobTtlSeconds", `${label}.budgets`), - capabilityJobBackoffLimit: integerField(budgets, "capabilityJobBackoffLimit", `${label}.budgets`), - }, - commands: { - plan: parseCommand(recordField(commands, "plan", `${label}.commands`), `${label}.commands.plan`), - status: parseCommand(recordField(commands, "status", `${label}.commands`), `${label}.commands.status`), - trigger: parseCommand(recordField(commands, "trigger", `${label}.commands`), `${label}.commands.trigger`), - events: parseCommand(recordField(commands, "events", `${label}.commands`), `${label}.commands.events`), - logs: parseCommand(recordField(commands, "logs", `${label}.commands`), `${label}.commands.logs`), - }, - nativeStatus: parseNativeStatus(nativeStatus, `${label}.nativeStatus`), - closeoutChecks: stringArrayField(closeout, "checks", `${label}.closeout`), - }; -} - -function parseNativeStatus(root: Record, label: string): NativeStatusSpec { - const source = recordField(root, "source", label); - const tekton = asOptionalRecord(root.tekton); - const argo = asOptionalRecord(root.argo); - const runtime = asOptionalRecord(root.runtime); - return { - source: { - gitMirrorReadUrl: stringField(source, "gitMirrorReadUrl", `${label}.source`), - gitMirrorNamespace: stringField(source, "gitMirrorNamespace", `${label}.source`), - gitMirrorDeployment: stringField(source, "gitMirrorDeployment", `${label}.source`), - repoPath: stringField(source, "repoPath", `${label}.source`), - }, - tekton: tekton === null - ? null - : { - namespace: stringField(tekton, "namespace", `${label}.tekton`), - pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${label}.tekton`), - }, - argo: argo === null - ? null - : { - namespace: stringField(argo, "namespace", `${label}.argo`), - application: stringField(argo, "application", `${label}.argo`), - }, - runtime: runtime === null - ? null - : { - namespace: stringField(runtime, "namespace", `${label}.runtime`), - workloads: arrayField(runtime, "workloads", `${label}.runtime`).map((item, index) => parseNativeWorkload(item, `${label}.runtime.workloads[${index}]`)), - }, - }; -} - -function parseNativeWorkload(value: unknown, label: string): NativeWorkloadSpec { - const root = asRecord(value, label); - const kind = stringField(root, "kind", label); - if (kind !== "Deployment" && kind !== "StatefulSet") throw new Error(`${label}.kind must be Deployment or StatefulSet`); - const sourceCommit = asOptionalRecord(root.sourceCommit) ?? {}; - return { - kind, - name: stringField(root, "name", label), - sourceCommit: { - labels: optionalStringArrayField(sourceCommit, "labels", `${label}.sourceCommit`), - annotations: optionalStringArrayField(sourceCommit, "annotations", `${label}.sourceCommit`), - podLabels: optionalStringArrayField(sourceCommit, "podLabels", `${label}.sourceCommit`), - podAnnotations: optionalStringArrayField(sourceCommit, "podAnnotations", `${label}.sourceCommit`), - env: optionalStringArrayField(sourceCommit, "env", `${label}.sourceCommit`), - }, - }; -} - -function parseCommand(root: Record, label: string): CommandSpec { - return { - argv: stringArrayField(root, "argv", label), - timeoutSeconds: integerField(root, "timeoutSeconds", label), - }; -} - -function optionalStringArrayField(root: Record, key: string, label: string): string[] { - return root[key] === undefined ? [] : stringArrayField(root, key, label); -} - -function stringMap(root: Record, label: string): Record { - const result: Record = {}; - for (const [key, value] of Object.entries(root)) { - if (typeof value !== "string" || value.length === 0) throw new Error(`${label}.${key} must be a non-empty string`); - result[key] = value; - } - return result; -} - -function buildPlan(registry: BranchFollowerRegistry, options: ParsedOptions): Record { - const selected = selectFollowers(registry, options, { includeDisabled: true }); - const followers = selected.map((follower) => { - const branchValue = safeResolveString(follower.source.branchRef); - const graph = configRefGraph([ - { id: "source.branch", ref: follower.source.branchRef }, - { id: "source.authority", ref: follower.source.authorityRef }, - { id: "source.snapshot", ref: follower.source.snapshotRef }, - ...Object.entries(follower.target.configRefs).map(([id, ref]) => ({ id: `target.${id}`, ref })), - ]); - const warnings: string[] = []; - if (branchValue !== null && branchValue !== follower.source.branch) warnings.push(`source.branch ${follower.source.branch} differs from ${follower.source.branchRef} value ${branchValue}`); - return { - id: follower.id, - enabled: follower.enabled, - adapter: follower.adapter, - description: follower.description, - source: { - repository: follower.source.repository, - branch: follower.source.branch, - branchRef: follower.source.branchRef, - resolvedBranch: branchValue, - snapshotPrefix: follower.source.snapshotPrefix, - }, - target: follower.target, - budgets: follower.budgets, - commands: redactCommands(follower), - nativeStatus: nativeStatusPlan(follower.nativeStatus), - closeoutChecks: follower.closeoutChecks, - configRefGraph: graph, - warnings, - }; - }); - return { - ok: true, - action: "plan", - spec: `${SPEC_REF} ${SPEC_VERSION}`, - registry: registrySummary(registry), - hostWorktreeAuthority: false, - sourceAuthority: { - mode: registry.controller.source.sourceAuthority.mode, - resolver: registry.controller.source.sourceAuthority.resolver, - allowHostGit: registry.controller.source.sourceAuthority.allowHostGit, - allowHostWorkspace: registry.controller.source.sourceAuthority.allowHostWorkspace, - allowGithubDirectInPipeline: registry.controller.source.sourceAuthority.allowGithubDirectInPipeline, - }, - controller: { - namespace: registry.controller.namespace, - kubeRoute: registry.controller.kubeRoute, - deploymentName: registry.controller.deploymentName, - stateConfigMapName: registry.controller.stateConfigMapName, - leaseName: registry.controller.leaseName, - image: registry.controller.image, - loop: registry.controller.loop, - budgets: registry.controller.budgets, - }, - followers, - next: { - apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", - status: "bun scripts/cli.ts cicd branch-follower status", - dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", - }, - }; -} - -async function applyController(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { - const manifests = renderControllerManifests(registry); - const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; - const manifestBase64 = Buffer.from(manifestYaml, "utf8").toString("base64"); - const waitSeconds = options.timeoutSeconds ?? registry.controller.budgets.applyWaitSeconds; - const script = [ - "set -eu", - "tmp=$(mktemp)", - "base64 -d >\"$tmp\" <<'UNIDESK_CICD_BRANCH_FOLLOWER_MANIFEST_B64'", - manifestBase64, - "UNIDESK_CICD_BRANCH_FOLLOWER_MANIFEST_B64", - options.dryRun - ? `kubectl apply --dry-run=server --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp"` - : `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp"`, - !options.dryRun && options.wait - ? `kubectl -n ${shQuote(registry.controller.namespace)} rollout status deploy/${shQuote(registry.controller.deploymentName)} --timeout=${waitSeconds}s` - : "true", - `kubectl -n ${shQuote(registry.controller.namespace)} get deploy/${shQuote(registry.controller.deploymentName)} cm/${shQuote(registry.controller.configMapName)} cm/${shQuote(registry.controller.stateConfigMapName)} lease/${shQuote(registry.controller.leaseName)} -o wide 2>/dev/null || true`, - ].join("\n"); - const result = runKubeScript(registry, options, script, "", (waitSeconds + 15) * 1000); - return { - ok: result.exitCode === 0, - action: "apply", - dryRun: options.dryRun, - wait: options.wait, - registry: registrySummary(registry), - objects: manifests.map((item) => objectRef(item)), - manifestSha256: createHash("sha256").update(manifestYaml).digest("hex"), - controller: { - namespace: registry.controller.namespace, - route: registry.controller.kubeRoute, - deploymentName: registry.controller.deploymentName, - stateConfigMapName: registry.controller.stateConfigMapName, - leaseName: registry.controller.leaseName, - hostWorktreeMounted: false, - sourceMode: "k8s-git-mirror-to-emptyDir", - }, - command: commandCompact(result, options), - next: { - status: "bun scripts/cli.ts cicd branch-follower status", - logs: `bun scripts/cli.ts cicd branch-follower logs --follower ${registry.followers[0]?.id ?? ""}`, - dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", - }, - }; -} - -async function buildStatus(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { - let k8s = readK8sState(registry, options); - const wantsLive = options.live || (!options.noLive && Object.keys(k8s.stateByFollower).length === 0); - const refresh = wantsLive && !options.inCluster ? runControllerReconcileJob(registry, options, { dryRun: true, wait: true, recordState: true }) : null; - if (refresh !== null) k8s = readK8sState(registry, options); - const shouldLive = wantsLive && options.inCluster; - const selected = selectFollowers(registry, options, { includeDisabled: true }); - const followers = []; - for (const follower of selected) { - const stored = k8s.stateByFollower[follower.id] ?? {}; - const live = shouldLive && follower.enabled ? await readAdapterStatus(registry, follower, options) : null; - followers.push(mergeFollowerStatus(registry, follower, stored, live, shouldLive)); - } - return { - ok: k8s.ok && followers.every((item) => item.ok !== false), - action: "status", - live: shouldLive, - registry: registrySummary(registry), - controller: controllerStatusSummary(registry, k8s), - followers, - refresh, - errors: k8s.errors, - next: { - apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait", - liveStatus: "bun scripts/cli.ts cicd branch-follower status --live", - dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run", - }, - }; -} - -async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { - if (!options.inCluster) { - const refresh = runControllerReconcileJob(registry, options, { dryRun: options.dryRun, wait: true, recordState: true }); - const k8s = readK8sState(registry, options); - const selected = selectFollowers(registry, options, { includeDisabled: false }); - return { - ok: refresh.ok, - action: "run-once", - dryRun: options.dryRun, - confirm: options.confirm, - wait: true, - controller: false, - execution: "k8s-native-reconcile-job", - registry: registrySummary(registry), - job: refresh, - followers: selected.map((follower) => mergeFollowerStatus(registry, follower, k8s.stateByFollower[follower.id] ?? {}, null, false)), - warnings: refresh.ok ? [] : [`reconcile job failed: ${refresh.message}`], - next: { - status: "bun scripts/cli.ts cicd branch-follower status", - liveStatus: "bun scripts/cli.ts cicd branch-follower status --live", - }, - }; - } - const selected = selectFollowers(registry, options, { includeDisabled: false }); - const previous = readK8sState(registry, options); - const results: FollowerState[] = []; - const stateWriteWarnings: string[] = []; - const stateWrites: Record[] = []; - for (const follower of selected) { - const oldState = previous.stateByFollower[follower.id] ?? {}; - const live = await readAdapterStatus(registry, follower, options); - const state = await decideAndMaybeTrigger(registry, follower, oldState, live, options); - if (!options.dryRun || options.recordState) { - const write = writeFollowerState(registry, state, options); - const writeSummary = stateWriteSummary(follower.id, write); - stateWrites.push(writeSummary); - if (write.exitCode !== 0) { - const warning = `state write failed for ${follower.id}: ${tailText(write.stderr || write.stdout, 300)}`; - state.warnings.push(warning); - stateWriteWarnings.push(warning); - } - } - results.push(state); - } - return { - ok: results.every((item) => item.phase !== "Failed" && item.phase !== "Blocked"), - action: "run-once", - dryRun: options.dryRun, - confirm: options.confirm, - wait: options.wait, - controller: options.inCluster, - registry: registrySummary(registry), - followers: results, - stateWrites, - warnings: stateWriteWarnings, - next: { - status: "bun scripts/cli.ts cicd branch-follower status", - liveStatus: "bun scripts/cli.ts cicd branch-follower status --live", - }, - }; -} - -function cleanupState(registry: BranchFollowerRegistry, options: ParsedOptions): Record { - const selected = selectFollowers(registry, options, { includeDisabled: true }); - const ids = selected.map((follower) => follower.id); - const before = kubeConfigMapFollowerState(registry, options); - const presentKeys = ids.filter((id) => before.stateByFollower[id] !== undefined); - const command = options.confirm ? removeFollowerStateKeys(registry, options, ids) : null; - return { - ok: command === null ? before.ok : command.exitCode === 0, - action: "cleanup-state", - dryRun: !options.confirm, - confirm: options.confirm, - execution: "k8s-state-configmap-key-delete", - registry: registrySummary(registry), - controller: { - namespace: registry.controller.namespace, - route: registry.controller.kubeRoute, - stateConfigMapName: registry.controller.stateConfigMapName, - }, - followers: ids.map((id) => ({ - id, - statePresent: before.stateByFollower[id] !== undefined, - cleanup: options.confirm ? "deleted-if-present" : "would-delete-if-present", - })), - stateConfigMapPresent: before.present, - presentKeys, - parsedDownstreamCliOutput: false, - command: command === null ? null : commandCompact(command, options), - errors: before.ok ? [] : [before.error], - next: { - status: "bun scripts/cli.ts cicd branch-follower status", - runOnce: options.followerId === null - ? "bun scripts/cli.ts cicd branch-follower run-once --all --confirm --wait" - : `bun scripts/cli.ts cicd branch-follower run-once --follower ${options.followerId} --confirm --wait`, - }, - }; -} - -async function runFollowerDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise> { - if (options.followerId === null) { - return { - ok: true, - action: options.action, - message: "select one follower to inspect native Kubernetes status objects", - followers: registry.followers.map((follower) => ({ - id: follower.id, - adapter: follower.adapter, - statusAuthority: "k8s-native", - nativeStatus: nativeStatusPlan(follower.nativeStatus), - })), - }; - } - const follower = registry.followers.find((item) => item.id === options.followerId); - if (follower === undefined) throw new Error(`unknown follower ${options.followerId}`); - if (!options.inCluster) { - const refresh = runControllerReconcileJob(registry, options, { dryRun: true, wait: true, recordState: true }); - const k8s = readK8sState(registry, options); - const stored = k8s.stateByFollower[follower.id] ?? {}; - const storedSource = asOptionalRecord(stored.source); - const storedTarget = asOptionalRecord(stored.target); - const command = asOptionalRecord(stored.command); - const native = asOptionalRecord(command?.payload); - return { - ok: refresh.ok && k8s.ok && Object.keys(stored).length > 0, - action: options.action, - follower: follower.id, - adapter: follower.adapter, - statusAuthority: "k8s-native-state-configmap", - parsedDownstreamCliOutput: false, - summary: { - phase: stringOrNull(stored.phase) ?? "Observed", - observedSha: stringOrNull(storedSource?.observedSha), - targetSha: stringOrNull(storedTarget?.targetSha), - pipelineRun: stringOrNull(stored.pipelineRun) ?? stringOrNull(stored.inFlightJob), - aligned: null, - message: stringOrNull(stored.decision) ?? "no controller state yet", - }, - native, - refresh, - errors: k8s.errors, - next: { - status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, - liveStatus: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --live`, - runOnceDryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`, - }, - }; - } - const live = await readAdapterStatus(registry, follower, options); - return { - ok: live.ok, - action: options.action, - follower: follower.id, - adapter: follower.adapter, - statusAuthority: "k8s-native", - parsedDownstreamCliOutput: false, - summary: { - phase: live.phase, - observedSha: live.observedSha, - targetSha: live.targetSha, - pipelineRun: live.pipelineRun, - aligned: live.aligned, - message: live.message, - }, - native: live.payload, - next: { - status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, - runOnceDryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`, - }, - }; -} - -async function decideAndMaybeTrigger( - registry: BranchFollowerRegistry, - follower: FollowerSpec, - previous: Record, - live: AdapterSummary, - options: ParsedOptions, -): Promise { - const warnings: string[] = []; - if (!live.ok) warnings.push(`status command failed: exitCode=${live.exitCode}${live.timedOut ? " timedOut=true" : ""}`); - const observedSha = live.observedSha; - let targetSha = live.targetSha; - const previousPhase = stringOrNull(previous.phase); - const previousLastTriggered = previousPhase === "Failed" ? null : stringOrNull(previous.lastTriggeredSha); - const previousInFlight = stringOrNull(previous.inFlightJob); - const previousLastSucceeded = stringOrNull(previous.lastSucceededSha); - const previousObserved = stringOrNull(recordAt(previous, ["source"])?.observedSha); - const previousTarget = stringOrNull(recordAt(previous, ["target"])?.targetSha); - const superseded = previousInFlight !== null && previousObserved !== null && observedSha !== null && previousObserved !== observedSha; - const nativePipelineRunMatchesObserved = observedSha !== null - && live.pipelineRunPresent === true - && live.pipelineRun !== null - && live.pipelineRun === expectedPipelineRunName(follower, observedSha); - let phase: BranchFollowerPhase; - let decision: string; - let triggerCommand: Record | undefined; - let inFlightJob: string | null = live.inFlightJob; - let lastTriggeredSha = live.lastTriggeredSha ?? previousLastTriggered ?? (nativePipelineRunMatchesObserved ? observedSha : null); - let lastSucceededSha = live.lastSucceededSha ?? previousLastSucceeded; - if (targetSha === null && observedSha !== null && previousLastSucceeded === observedSha && previousTarget === observedSha) targetSha = observedSha; - - if (!follower.enabled) { - phase = "Skipped"; - decision = "follower disabled"; - } else if (observedSha === null) { - phase = live.ok ? "Observed" : "Blocked"; - decision = "status did not expose an observed source sha; adapter trigger-current remains the dedupe authority"; - } else if (superseded) { - phase = "Superseded"; - decision = `previous in-flight sha ${shortSha(previousObserved)} was superseded by ${shortSha(observedSha)}`; - } else if (targetSha !== null && targetSha === observedSha && live.aligned === true) { - phase = "Noop"; - decision = "target already matches observed source sha"; - inFlightJob = null; - lastSucceededSha = observedSha; - } else if (lastTriggeredSha !== null && lastTriggeredSha === observedSha && live.pipelineRunPresent === true) { - phase = "ClosingOut"; - decision = targetSha === observedSha - ? "target sha matches observed source sha but native closeout gates are not complete" - : "same sha was already triggered; use status/events/logs for closeout"; - } else if (lastTriggeredSha !== null && lastTriggeredSha === observedSha) { - phase = "PendingTrigger"; - decision = "same sha has a trigger record but no PipelineRun is present; retrying trigger"; - } else if (targetSha !== null && targetSha === observedSha) { - phase = "ClosingOut"; - decision = "target sha matches observed source sha but native closeout gates are not complete"; - } else { - phase = "PendingTrigger"; - decision = targetSha === null - ? "target sha is unknown; trigger-current adapter will dedupe by source snapshot" - : `observed ${shortSha(observedSha)} differs from target ${shortSha(targetSha)}`; - } - - if (options.confirm && (phase === "PendingTrigger" || phase === "Superseded" || (phase === "Observed" && observedSha !== null))) { - const trigger = await executeTrigger(registry, follower, observedSha, options); - triggerCommand = trigger.command; - phase = trigger.ok ? (options.wait ? "ClosingOut" : "Triggering") : "Failed"; - decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}: ${redactText(trigger.message).slice(0, 220)}`; - inFlightJob = trigger.jobId ?? live.inFlightJob; - lastTriggeredSha = observedSha; - if (trigger.ok && options.wait && trigger.completed) { - phase = "Succeeded"; - decision = `trigger completed for ${shortSha(observedSha)}`; - inFlightJob = null; - targetSha = observedSha; - lastSucceededSha = observedSha; - } - if (!trigger.ok) warnings.push(trigger.message); - } - if (options.confirm && options.wait && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) { - const closeout = await waitNativeFollowerCloseout(registry, follower, observedSha, options, options.timeoutSeconds ?? follower.budgets.endToEndSeconds); - triggerCommand = closeoutOnlyCommand(follower, live.pipelineRun, observedSha, closeout); - if (closeout.completed) { - phase = "Succeeded"; - decision = `closeout completed for ${shortSha(observedSha)}`; - inFlightJob = null; - targetSha = observedSha; - lastSucceededSha = observedSha; - } else { - decision = closeout.timedOut - ? `closeout did not converge for ${shortSha(observedSha)} within budget` - : `closeout remains pending for ${shortSha(observedSha)}`; - warnings.push(decision); - } - } - - if (options.dryRun && phase === "PendingTrigger") decision = `${decision}; dry-run did not trigger`; - if (shouldRefreshAutomaticCloseout(follower, observedSha, live, phase, options)) { - const refresh = runNativeArgoRefresh(follower.nativeStatus.argo as NonNullable); - if (refresh.exitCode !== 0) warnings.push(`argo refresh failed: ${redactText(tailText(refresh.stderr || refresh.stdout, 300))}`); - } - const statePipelineRun = stringOrNull(triggerCommand?.pipelineRun) ?? live.pipelineRun; - - return { - id: follower.id, - adapter: follower.adapter, - enabled: follower.enabled, - phase, - source: { - repository: follower.source.repository, - branch: follower.source.branch, - branchRef: follower.source.branchRef, - snapshotPrefix: follower.source.snapshotPrefix, - observedSha, - }, - target: { - node: follower.target.node, - lane: follower.target.lane, - namespace: follower.target.namespace, - sentinel: follower.target.sentinel, - targetSha, - }, - lastTriggeredSha, - lastSucceededSha, - pipelineRun: statePipelineRun, - inFlightJob, - budgetSource: follower.budgets, - controller: { - mode: options.inCluster ? "k8s-controller" : "local-cli", - stateConfigMap: registry.controller.stateConfigMapName, - leaseName: registry.controller.leaseName, - }, - decision, - dryRun: options.dryRun, - updatedAt: new Date().toISOString(), - timings: buildFollowerTimings(follower, live, triggerCommand, asOptionalRecord(previous.timings), phase), - warnings, - next: followerNextCommands(follower), - command: triggerCommand ?? { - status: live.command, - exitCode: live.exitCode, - timedOut: live.timedOut, - payload: live.payload, - }, - }; -} - -function shouldRefreshAutomaticCloseout( - follower: FollowerSpec, - observedSha: string | null, - live: AdapterSummary, - phase: BranchFollowerPhase, - options: ParsedOptions, -): boolean { - if (!options.inCluster || !options.confirm || options.wait || options.dryRun) return false; - if (phase !== "ClosingOut" || observedSha === null || follower.nativeStatus.argo === null) return false; - const payload = asOptionalRecord(live.payload); - const tekton = asOptionalRecord(payload?.tekton); - if (tekton?.succeeded !== true) return false; - const argo = asOptionalRecord(payload?.argo); - const runtime = asOptionalRecord(payload?.runtime); - const argoReady = argo?.ready === true; - const runtimeAligned = runtime?.aligned === true; - return !argoReady || !runtimeAligned || live.targetSha !== observedSha; -} - -async function executeTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions): Promise { - const spec = follower.commands.trigger; - const timeoutSeconds = options.timeoutSeconds ?? spec.timeoutSeconds; - if (follower.adapter === "hwlab-node-runtime" && options.inCluster) { - return await executeNativeHwlabNodeTrigger(registry, follower, observedSha, options, timeoutSeconds); - } - if (follower.adapter === "agentrun-yaml-lane" && options.inCluster) { - return await executeNativeAgentRunTrigger(registry, follower, observedSha, options, timeoutSeconds); - } - if (follower.adapter === "web-probe-sentinel-cicd" && options.inCluster) { - return await executeNativeSentinelTrigger(registry, follower, observedSha, options, timeoutSeconds); - } - if (!options.wait && !options.inCluster) { - const job = startJob(`cicd_branch_follower_${safeJobSegment(follower.id)}`, spec.argv, `Trigger ${follower.id} for observed sha ${observedSha ?? "unknown"}`); - return { - ok: true, - completed: false, - message: `started async job ${job.id}`, - jobId: job.id, - command: { - mode: "async-job", - argv: spec.argv, - jobId: job.id, - status: `bun scripts/cli.ts job status ${job.id}`, - }, - }; - } - const result = runCommand(spec.argv, repoRoot, { timeoutMs: timeoutSeconds * 1000 }); - return { - ok: result.exitCode === 0, - completed: result.exitCode === 0 && options.wait, - message: result.exitCode === 0 ? "trigger command completed" : tailText(result.stderr || result.stdout, 500), - jobId: null, - command: commandCompact(result, options), - }; -} - -async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions, timeoutSeconds: number): Promise { - if (observedSha === null) { - return nativeTriggerError(follower, "native HWLAB trigger requires observed source sha", "observed-sha-missing"); - } - if (follower.nativeStatus.tekton === null) { - return nativeTriggerError(follower, "native HWLAB trigger requires Tekton nativeStatus", "tekton-native-status-missing"); - } - const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); - const pipelineRun = nodeRuntimePipelineRunName(spec, observedSha); - const namespace = follower.nativeStatus.tekton.namespace; - const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun); - const startedAt = Date.now(); - const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds)); - if (sync !== null && !sync.result.ok) { - return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt); - } - const refresh = runNativeHwlabControlPlaneRefresh(registry, follower, spec, observedSha, Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.controlPlaneRefreshSeconds), nativeCapabilityJobName(follower.id, "control-plane-refresh", observedSha)); - if (!refresh.result.ok) { - return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt); - } - const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets); - const payload = parseJsonObject(result.stdout) ?? {}; - const pipelineRunCompleted = payload.completed === true; - const failed = payload.failed === true || result.exitCode !== 0; - const flush = !failed && options.wait && pipelineRunCompleted - ? runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds)) - : null; - if (flush !== null && !flush.result.ok) { - return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native git-mirror flush failed", startedAt); - } - const closeout = !failed && options.wait && pipelineRunCompleted - ? await waitNativeFollowerCloseout(registry, follower, observedSha, options, remainingSeconds(startedAt, timeoutSeconds)) - : null; - return nativeTektonTriggerResult({ - follower, - observedSha, - namespace, - pipelineRun, - stageRef: `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`, - wait: options.wait, - result, - startedAt, - payload: { - ...payload, - nativeCapabilities: { - gitMirrorSync: sync === null ? null : sync.result, - controlPlaneRefresh: refresh.result, - gitMirrorFlush: flush === null ? null : flush.result, - }, - }, - closeout, - successMessage: `native HWLAB PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}`, - }); -} - -async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions, timeoutSeconds: number): Promise { - if (observedSha === null) { - return nativeTriggerError(follower, "native AgentRun trigger requires observed source sha", "observed-sha-missing"); - } - if (follower.nativeStatus.tekton === null) { - return nativeTriggerError(follower, "native AgentRun trigger requires Tekton nativeStatus", "tekton-native-status-missing"); - } - const { configPath, spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); - const startedAt = Date.now(); - const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`; - const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`; - const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds)); - if (sync !== null && !sync.result.ok) { - return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt); - } - const buildJob = `${jobPrefix}-build-${observedSha.slice(0, 12)}`.slice(0, 63); - const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), spec.deployment.manager.imageBuild.timeoutSeconds), "buildkit", registry.controller.budgets); - const buildPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(build.logsTail) ?? "" }); - const digest = stringOrNull(buildPayload.digest); - const envIdentity = stringOrNull(buildPayload.envIdentity); - if (!build.ok || digest === null || envIdentity === null) { - return nativeK8sStageFailure(follower, observedSha, "image-build", buildJob, build, buildPayload, "native AgentRun image build failed", startedAt); - } - const image = agentRunImageArtifact(spec, { sourceCommit: observedSha, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" }); - const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: observedSha, image }); - const publishJob = `${jobPrefix}-gitops-${observedSha.slice(0, 12)}`.slice(0, 63); - const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish", registry.controller.budgets); - const publishPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(publish.logsTail) ?? "" }); - if (!publish.ok || publishPayload.ok === false || stringOrNull(publishPayload.gitopsCommit) === null) { - return nativeK8sStageFailure(follower, observedSha, "gitops-publish", publishJob, publish, publishPayload, "native AgentRun GitOps publish failed", startedAt); - } - const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds)); - if (flush !== null && !flush.result.ok) { - return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native AgentRun git-mirror flush failed", startedAt); - } - const pipelineRun = agentRunPipelineRunName(spec, observedSha); - const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets); - const tektonPayload = parseJsonObject(tektonResult.stdout) ?? {}; - const pipelineRunCompleted = tektonPayload.completed === true; - const failed = tektonPayload.failed === true || tektonResult.exitCode !== 0; - const closeout = !failed && options.wait && pipelineRunCompleted - ? await waitNativeFollowerCloseout(registry, follower, observedSha, options, remainingSeconds(startedAt, timeoutSeconds)) - : null; - const trigger = nativeTektonTriggerResult({ - follower, - observedSha, - namespace: follower.nativeStatus.tekton.namespace, - pipelineRun, - stageRef, - wait: options.wait, - result: tektonResult, - startedAt, - payload: { - ...tektonPayload, - agentrun: { - configPath, - gitMirrorSync: sync === null ? null : { jobName: sync.jobName, payload: sync.result }, - imageBuild: { jobName: buildJob, result: build, payload: buildPayload }, - gitopsPublish: { jobName: publishJob, result: publish, payload: publishPayload }, - gitMirrorFlush: flush === null ? null : { jobName: flush.jobName, payload: flush.result }, - }, - }, - closeout, - successMessage: `native AgentRun PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}`, - }); - return trigger; -} - -function nativeTriggerError(follower: FollowerSpec, message: string, reason: string): TriggerResult { - return { - ok: false, - completed: false, - message, - jobId: null, - command: { mode: "k8s-native-tekton", adapter: follower.adapter, ok: false, reason, parsedDownstreamCliOutput: false }, - }; -} - -function nativeK8sStageFailure( - follower: FollowerSpec, - observedSha: string, - phase: string, - jobName: string, - job: NativeK8sJobResult, - payload: Record, - message: string, - startedAt?: number, -): TriggerResult { - const detail = [ - message, - job.timedOut ? "timedOut=true" : null, - job.conditionReason === null ? null : `reason=${job.conditionReason}`, - job.conditionMessage === null ? null : `condition=${job.conditionMessage}`, - job.logsTail === null ? null : `logs=${tailText(job.logsTail, 500)}`, - ].filter((item): item is string => item !== null).join("; "); - return { - ok: false, - completed: false, - message: detail, - jobId: jobName, - command: { - mode: "k8s-native-job", - adapter: follower.adapter, - phase, - jobName, - sourceCommit: observedSha, - ok: false, - startedAt: startedAt === undefined ? null : new Date(startedAt).toISOString(), - finishedAt: new Date().toISOString(), - elapsedMs: startedAt === undefined ? job.elapsedMs : Date.now() - startedAt, - payload, - job, - statusAuthority: "kubernetes-api-serviceaccount", - parsedDownstreamCliOutput: false, - }, - }; -} - -function runNativeGitMirrorStage(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string, action: "sync" | "flush", timeoutSeconds: number, jobKey = observedSha): { jobName: string; namespace: string; result: NativeK8sJobResult } | null { - const job = nativeGitMirrorJobForFollower(follower, jobKey, action); - if (job === null) return null; - const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action, registry.controller.budgets); - return { jobName: job.jobName, namespace: job.namespace, result }; -} - -function nativeGitMirrorJobForFollower( - follower: FollowerSpec, - observedSha: string, - action: "sync" | "flush", -): { namespace: string; jobName: string; manifest: Record } | null { - const jobName = nativeCapabilityJobName(follower.id, action, observedSha); - if (follower.adapter === "hwlab-node-runtime") { - const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); - const mirror = nodeRuntimeGitMirrorTarget(spec); - return { - namespace: mirror.namespace, - jobName, - manifest: nodeRuntimeGitMirrorJobManifest(mirror, action, jobName), - }; - } - if (follower.adapter === "agentrun-yaml-lane") { - const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); - return { - namespace: spec.gitMirror.namespace, - jobName, - manifest: yamlLaneGitMirrorJobManifest(spec, action, jobName), - }; - } - return null; -} - -function nativeTektonTriggerResult(input: { - follower: FollowerSpec; - observedSha: string; - namespace: string; - pipelineRun: string; - stageRef: string; - wait: boolean; - result: CommandResult; - startedAt: number; - payload: Record; - closeout: NativeCloseoutWaitResult | null; - successMessage: string; -}): TriggerResult { - const pipelineRunCompleted = input.payload.completed === true; - const failed = input.payload.failed === true || input.result.exitCode !== 0; - const stillRunning = input.payload.stillRunning === true || input.payload.timedOutWait === true; - const message = failed - ? nativeTektonFailureText(input.payload, input.result) - : input.closeout?.completed === true - ? input.successMessage - : input.closeout?.timedOut === true - ? `native PipelineRun ${input.pipelineRun} succeeded but runtime closeout did not converge within budget` - : pipelineRunCompleted - ? `native PipelineRun ${input.pipelineRun} succeeded; runtime closeout remains k8s-native` - : stillRunning - ? `native PipelineRun ${input.pipelineRun} is still running; query status/events/logs for closeout` - : `native PipelineRun ${input.pipelineRun} submitted`; - const ok = !failed && (input.closeout === null || input.closeout.completed === true); - const finishedAt = failed || input.result.timedOut || input.closeout?.completed === true || input.closeout?.timedOut === true - ? new Date().toISOString() - : null; - return { - ok, - completed: input.closeout?.completed === true, - message, - jobId: input.pipelineRun, - command: { - mode: "k8s-native-tekton", - adapter: input.follower.adapter, - namespace: input.namespace, - pipelineRun: input.pipelineRun, - sourceCommit: input.observedSha, - sourceStageRef: input.stageRef, - wait: input.wait, - pipelineRunCompleted, - stillRunning, - startedAt: new Date(input.startedAt).toISOString(), - finishedAt, - elapsedMs: Date.now() - input.startedAt, - closeout: input.closeout, - statusAuthority: "kubernetes-api-serviceaccount", - parsedDownstreamCliOutput: false, - payload: input.payload, - exitCode: input.result.exitCode, - timedOut: input.result.timedOut, - stderrTail: failed ? redactText(tailText(input.result.stderr, 1000)) : "", - }, - }; -} - -function nativeTektonFailureText(payload: Record, result: CommandResult): string { - return [ - stringOrNull(payload.message), - stringOrNull(payload.conditionReason), - stringOrNull(payload.conditionMessage), - payload.failed === true ? "PipelineRun failed" : null, - result.timedOut ? "timedOut=true" : null, - ].filter((item): item is string => item !== null && item.length > 0).join("; ") - || "native PipelineRun failed; query events/logs for structured details"; -} - -function remainingSeconds(startedAt: number, timeoutSeconds: number): number { - return Math.max(1, timeoutSeconds - Math.ceil((Date.now() - startedAt) / 1000)); -} - -async function executeNativeSentinelTrigger(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string | null, options: ParsedOptions, timeoutSeconds: number): Promise { - if (observedSha === null) { - return { - ok: false, - completed: false, - message: "native sentinel trigger requires observed source sha", - jobId: null, - command: { mode: "k8s-native-tekton", ok: false, reason: "observed-sha-missing", parsedDownstreamCliOutput: false }, - }; - } - if (follower.target.sentinel === null) { - return { - ok: false, - completed: false, - message: "native sentinel trigger requires target.sentinel", - jobId: null, - command: { mode: "k8s-native-tekton", ok: false, reason: "target-sentinel-missing", parsedDownstreamCliOutput: false }, - }; - } - const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); - const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`; - const state = loadSentinelCicdState(spec, follower.target.sentinel, timeoutSeconds, "cached", { - commit: observedSha, - stageRef, - mirrorCommit: observedSha, - sourceAuthority: "git-mirror-snapshot", - }); - const pipelineRun = sentinelPipelineRunName(state, false); - const namespace = stringField(recordField(state.cicd, "builder", `${follower.id}.sentinel.cicd`), "namespace", `${follower.id}.sentinel.cicd.builder`); - const manifest = sentinelPublishPipelineRunManifest(state, pipelineRun, true); - const startedAt = Date.now(); - const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds, registry.controller.budgets); - const payload = parseJsonObject(result.stdout) ?? {}; - const pipelineRunCompleted = payload.completed === true; - const failed = payload.failed === true || result.exitCode !== 0; - const stillRunning = payload.stillRunning === true || payload.timedOutWait === true; - const remainingSeconds = Math.max(1, timeoutSeconds - Math.ceil((Date.now() - startedAt) / 1000)); - const closeout = !failed && options.wait && pipelineRunCompleted - ? await waitNativeSentinelCloseout(registry, follower, observedSha, options, remainingSeconds) - : null; - const message = failed - ? tailText(result.stderr || stringOrNull(payload.message) || result.stdout, 500) - : closeout?.completed === true - ? `native sentinel PipelineRun ${pipelineRun} succeeded and runtime reached ${shortSha(observedSha)}` - : closeout?.timedOut === true - ? `native sentinel PipelineRun ${pipelineRun} succeeded but runtime closeout did not converge within budget` - : pipelineRunCompleted - ? `native sentinel PipelineRun ${pipelineRun} succeeded; runtime closeout remains k8s-native` - : stillRunning - ? `native sentinel PipelineRun ${pipelineRun} is still running; query status/events/logs for closeout` - : `native sentinel PipelineRun ${pipelineRun} submitted`; - const ok = !failed && (closeout === null || closeout.completed === true); - const finishedAt = failed || result.timedOut || closeout?.completed === true || closeout?.timedOut === true - ? new Date().toISOString() - : null; - return { - ok, - completed: closeout?.completed === true, - message, - jobId: pipelineRun, - command: { - mode: "k8s-native-tekton", - adapter: follower.adapter, - namespace, - pipelineRun, - sourceCommit: observedSha, - sourceStageRef: stageRef, - wait: options.wait, - pipelineRunCompleted, - stillRunning, - startedAt: new Date(startedAt).toISOString(), - finishedAt, - elapsedMs: Date.now() - startedAt, - closeout, - statusAuthority: "kubernetes-api-serviceaccount", - parsedDownstreamCliOutput: false, - payload, - exitCode: result.exitCode, - timedOut: result.timedOut, - stderrTail: failed ? redactText(tailText(result.stderr, Math.min(options.tailBytes, 4000))) : "", - }, - }; -} - -async function waitNativeSentinelCloseout( - registry: BranchFollowerRegistry, - follower: FollowerSpec, - observedSha: string, - options: ParsedOptions, - timeoutSeconds: number, -): Promise { - const refreshResult = follower.nativeStatus.argo === null ? null : runNativeArgoRefresh(follower.nativeStatus.argo); - const startedAt = Date.now(); - const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000; - let polls = 0; - let latest: AdapterSummary | null = null; - let gitMirrorFlush: Record | null = null; - while (Date.now() <= deadline) { - polls += 1; - const remainingSeconds = Math.max(1, Math.ceil((deadline - Date.now()) / 1000)); - latest = await readAdapterStatus(registry, follower, { ...options, timeoutSeconds: Math.min(10, remainingSeconds) }); - const latestPayload = asOptionalRecord(latest.payload); - const latestGitMirror = asOptionalRecord(latestPayload?.gitMirror); - if (latest.observedSha === observedSha && asOptionalRecord(latestPayload?.tekton)?.succeeded === true && gitMirrorFlush === null && shouldFlushNativeGitMirrorDuringCloseout(follower, latestGitMirror)) { - const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", Math.min(remainingSeconds, follower.budgets.sourceSyncSeconds), stringOrNull(latestGitMirror?.localGitops) ?? observedSha); - gitMirrorFlush = flush === null ? null : { - jobName: flush.jobName, - namespace: flush.namespace, - result: compactNativeK8sJobResult(flush.result), - }; - if (flush !== null && !flush.result.ok) { - return { - ok: false, - completed: false, - timedOut: flush.result.timedOut, - polls, - elapsedMs: Date.now() - startedAt, - refresh: refreshResult === null ? null : commandCompact(refreshResult, options), - gitMirrorFlush, - summary: nativeCloseoutSummary(latest), - statusAuthority: "k8s-native", - parsedDownstreamCliOutput: false, - }; - } - continue; - } - if (latest.observedSha === observedSha && latest.aligned === true) { - return { - ok: true, - completed: true, - timedOut: false, - polls, - elapsedMs: Date.now() - startedAt, - refresh: refreshResult === null ? null : commandCompact(refreshResult, options), - gitMirrorFlush, - summary: nativeCloseoutSummary(latest), - statusAuthority: "k8s-native", - parsedDownstreamCliOutput: false, - }; - } - if (Date.now() + 2_000 > deadline) break; - runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); - } - return { - ok: false, - completed: false, - timedOut: true, - polls, - elapsedMs: Date.now() - startedAt, - refresh: refreshResult === null ? null : commandCompact(refreshResult, options), - gitMirrorFlush, - summary: latest === null ? null : nativeCloseoutSummary(latest), - statusAuthority: "k8s-native", - parsedDownstreamCliOutput: false, - }; -} - -function shouldFlushNativeGitMirrorDuringCloseout(follower: FollowerSpec, gitMirror: Record | null): boolean { - if (!nativeGitMirrorRequired(follower) || gitMirror === null) return false; - if (gitMirror.sourceSnapshotReady === false) return false; - return gitMirror.pendingFlush === true || gitMirror.githubInSync === false; -} - -function compactNativeK8sJobResult(result: NativeK8sJobResult): Record { - return { - ok: result.ok, - completed: result.completed, - failed: result.failed, - timedOut: result.timedOut, - created: result.created, - reused: result.reused, - jobName: result.jobName, - namespace: result.namespace, - polls: result.polls, - elapsedMs: result.elapsedMs, - conditionReason: result.conditionReason, - conditionMessage: result.conditionMessage, - statusAuthority: result.statusAuthority, - parsedDownstreamCliOutput: false, - }; -} - -async function waitNativeFollowerCloseout( - registry: BranchFollowerRegistry, - follower: FollowerSpec, - observedSha: string, - options: ParsedOptions, - timeoutSeconds: number, -): Promise { - return await waitNativeSentinelCloseout(registry, follower, observedSha, options, timeoutSeconds); -} - -function closeoutOnlyCommand(follower: FollowerSpec, pipelineRun: string | null, observedSha: string, closeout: NativeCloseoutWaitResult): Record { - return { - mode: "k8s-native-closeout", - adapter: follower.adapter, - pipelineRun, - sourceCommit: observedSha, - wait: true, - closeout, - finishedAt: closeout.completed || closeout.timedOut ? new Date().toISOString() : null, - elapsedMs: closeout.elapsedMs, - exitCode: closeout.completed ? 0 : 1, - timedOut: closeout.timedOut, - statusAuthority: "k8s-native", - parsedDownstreamCliOutput: false, - }; -} - -function nativeCloseoutSummary(live: AdapterSummary): Record { - const payload = asOptionalRecord(live.payload); - return { - ok: live.ok, - phase: live.phase, - observedSha: live.observedSha, - targetSha: live.targetSha, - aligned: live.aligned, - pipelineRun: live.pipelineRun, - pipelineRunPresent: live.pipelineRunPresent, - message: live.message, - gitMirror: asOptionalRecord(payload?.gitMirror), - tekton: asOptionalRecord(payload?.tekton), - taskRuns: compactTaskRunsPayload(asOptionalRecord(payload?.taskRuns)), - planArtifacts: compactPlanArtifactsPayload(asOptionalRecord(payload?.planArtifacts)), - argo: asOptionalRecord(payload?.argo), - runtime: asOptionalRecord(payload?.runtime), - errors: Array.isArray(payload?.errors) ? payload.errors.slice(0, 5) : [], - }; -} - -function runNativeArgoRefresh(argo: NonNullable): CommandResult { - const patchBase64 = Buffer.from(JSON.stringify({ - metadata: { - annotations: { - "argocd.argoproj.io/refresh": "hard", - }, - }, - }), "utf8").toString("base64"); - const script = [ - "set -eu", - `ARGO_NAMESPACE=${shQuote(argo.namespace)}`, - `ARGO_APPLICATION=${shQuote(argo.application)}`, - `PATCH_B64=${shQuote(patchBase64)}`, - "export ARGO_NAMESPACE ARGO_APPLICATION PATCH_B64", - "node <<'NODE_ARGO_REFRESH'", - "import { readFileSync } from 'node:fs';", - "import https from 'node:https';", - "const namespace = process.env.ARGO_NAMESPACE || '';", - "const application = process.env.ARGO_APPLICATION || '';", - "const patch = Buffer.from(process.env.PATCH_B64 || '', 'base64').toString('utf8');", - "const host = process.env.KUBERNETES_SERVICE_HOST;", - "const port = Number(process.env.KUBERNETES_SERVICE_PORT || '443');", - "const token = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/token', 'utf8').trim();", - "const ca = readFileSync('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt');", - "const path = `/apis/argoproj.io/v1alpha1/namespaces/${encodeURIComponent(namespace)}/applications/${encodeURIComponent(application)}`;", - "const req = https.request({ host, port, path, method: 'PATCH', ca, headers: { authorization: `Bearer ${token}`, 'content-type': 'application/merge-patch+json', 'content-length': Buffer.byteLength(patch) } }, (res) => {", - " let body = '';", - " res.setEncoding('utf8');", - " res.on('data', (chunk) => { body += chunk; });", - " res.on('end', () => {", - " if ((res.statusCode || 0) < 200 || (res.statusCode || 0) >= 300) { process.stderr.write(body || `kube api PATCH argo application status ${res.statusCode}`); process.exit(1); }", - " process.stdout.write(JSON.stringify({ ok: true, namespace, application, refresh: 'hard', statusAuthority: 'kubernetes-api-serviceaccount', parsedDownstreamCliOutput: false, valuesRedacted: true }));", - " });", - "});", - "req.on('error', (error) => { process.stderr.write(error?.message || String(error)); process.exit(1); });", - "req.write(patch);", - "req.end();", - "NODE_ARGO_REFRESH", - ].join("\n"); - return runCommand(["sh", "-lc", script], repoRoot, { timeoutMs: 10_000 }); -} - -async function readAdapterStatus(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Promise { - const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.statusSeconds; - const startedAt = Date.now(); - const sourceSync = runNativeSourceObservationSync(registry, follower, options, Math.min(timeoutSeconds, follower.budgets.sourceSyncSeconds)); - const sourceSyncDetail = sourceSync === null || sourceSync.result.ok ? null : redactText(tailText(sourceSync.result.conditionMessage ?? sourceSync.result.logsTail ?? "unknown", 800)); - const sourceSyncError = sourceSyncDetail === null ? null : `native source sync failed: ${sourceSyncDetail}`; - const bundle = readNativeObjectBundle(registry, follower, options, remainingSeconds(startedAt, timeoutSeconds), runKubeScript); - const observedSha = sourceSyncError === null ? stringOrNull(bundle.source?.commit) : null; - const runtimeTargetSha = runtimeTargetShaFromWorkloads(follower.nativeStatus.runtime, bundle.workloads); - const pipelineRunName = stringOrNull(asOptionalRecord(bundle.pipelineRun?.metadata)?.name) ?? expectedPipelineRunName(follower, observedSha); - const pipelineRunPresent = follower.nativeStatus.tekton === null ? null : bundle.pipelineRun !== null; - const pipelineSucceeded = pipelineRunSucceeded(bundle.pipelineRun); - const argoReady = follower.nativeStatus.argo === null ? null : argoApplicationReady(bundle.argoApplication); - const runtimeReady = follower.nativeStatus.runtime === null ? null : runtimeWorkloadsReady(follower.nativeStatus.runtime, bundle.workloads); - const gitMirrorRequired = nativeGitMirrorRequired(follower); - const gitMirrorReady = gitMirrorRequired ? nativeGitMirrorReady(bundle.gitMirror) : null; - const hasTektonGate = follower.nativeStatus.tekton !== null; - const hasRuntimeTarget = runtimeTargetSha !== null; - const requiresRuntimeTarget = follower.nativeStatus.runtime !== null; - const hasTargetEvidence = hasRuntimeTarget || (!requiresRuntimeTarget && hasTektonGate); - const pipelineGateOk = !hasTektonGate - || pipelineSucceeded === true - || (hasRuntimeTarget && runtimeTargetSha === observedSha && argoReady !== false && runtimeReady !== false); - const aligned = observedSha !== null - && hasTargetEvidence - && (requiresRuntimeTarget ? hasRuntimeTarget && runtimeTargetSha === observedSha : true) - && pipelineGateOk - && gitMirrorReady !== false - && argoReady !== false - && runtimeReady !== false; - const targetSha = hasRuntimeTarget && runtimeTargetSha === observedSha && aligned ? runtimeTargetSha : runtimeTargetSha; - const ok = bundle.ok && sourceSyncError === null; - const phase = inferPhase(ok, aligned, observedSha, targetSha, bundle.timedOut); - const errors = sourceSyncError === null ? bundle.errors : [sourceSyncError, ...bundle.errors]; - return { - ok, - command: "native:k8s-git-mirror+tekton+argocd+runtime", - exitCode: bundle.exitCode, - timedOut: bundle.timedOut, - observedSha, - targetSha, - lastTriggeredSha: null, - lastSucceededSha: aligned && observedSha !== null ? observedSha : null, - pipelineRun: pipelineRunName, - pipelineRunPresent, - inFlightJob: pipelineRunPresent === true && pipelineSucceeded === null && pipelineRunName !== null ? pipelineRunName : null, - aligned, - phase, - message: nativeStatusMessage(ok, phase, observedSha, targetSha, { - pipelineSucceeded, - gitMirrorReady, - argoReady, - runtimeReady, - errors, - }), - payload: { - source: bundle.source, - sourceSync: sourceSync === null ? null : sourceSync.result, - gitMirror: nativeGitMirrorSummary(bundle.gitMirror), - tekton: nativePipelineRunSummary(bundle.pipelineRun), - taskRuns: bundle.taskRuns, - planArtifacts: bundle.planArtifacts, - argo: nativeArgoSummary(bundle.argoApplication), - runtime: nativeRuntimeSummary(follower.nativeStatus.runtime, bundle.workloads, observedSha), - timings: { statusRead: { elapsedMs: bundle.elapsedMs, budgetSeconds: timeoutSeconds } }, - errors, - statusAuthority: "k8s-native", - parsedDownstreamCliOutput: false, - }, - stderrTail: bundle.stderrTail, - stdoutTail: bundle.stdoutTail, - }; -} - -function runNativeSourceObservationSync(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, timeoutSeconds: number): { jobName: string; namespace: string; result: NativeK8sJobResult } | null { - if (!options.inCluster) return null; - if (follower.adapter !== "hwlab-node-runtime" && follower.adapter !== "agentrun-yaml-lane") return null; - const bucket = Math.floor(Date.now() / (registry.controller.loop.intervalSeconds * 1000)).toString(36); - return runNativeGitMirrorStage(registry, follower, "source", "sync", timeoutSeconds, `source-${bucket}`); -} - -function inferPhase(ok: boolean, aligned: boolean | null, observedSha: string | null, targetSha: string | null, timedOut: boolean): BranchFollowerPhase { - if (!ok || timedOut) return "Blocked"; - if (aligned === true) return "Succeeded"; - if (observedSha !== null && targetSha !== null && observedSha === targetSha) return "ClosingOut"; - if (observedSha !== null) return "PendingTrigger"; - return "Observed"; -} - -function nativeStatusMessage(ok: boolean, phase: BranchFollowerPhase, observedSha: string | null, targetSha: string | null, gates: { pipelineSucceeded: boolean | null; gitMirrorReady: boolean | null; argoReady: boolean | null; runtimeReady: boolean | null; errors: string[] }): string { - if (!ok) return gates.errors[0] ?? "native status read failed"; - if (phase === "Noop" || phase === "Succeeded") return `target matches ${shortSha(observedSha)}`; - if (observedSha !== null) { - const gatesText = [ - gates.pipelineSucceeded === false ? "pipelineRun not successful" : null, - gates.gitMirrorReady === false ? "git-mirror not flushed" : null, - gates.argoReady === false ? "argo not healthy/synced" : null, - gates.runtimeReady === false ? "runtime not ready" : null, - ].filter((item): item is string => item !== null).join("; "); - if (targetSha !== null) return gatesText.length > 0 ? `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}; ${gatesText}` : `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}`; - return gatesText.length > 0 ? `observed ${shortSha(observedSha)}; ${gatesText}` : `observed ${shortSha(observedSha)} target unknown`; - } - return "k8s-native status did not expose observed source sha"; -} - -function expectedPipelineRunName(follower: FollowerSpec, observedSha: string | null): string | null { - if (observedSha === null || follower.nativeStatus.tekton === null) return null; - return `${follower.nativeStatus.tekton.pipelineRunPrefix}-${shortSha(observedSha)}`; -} - -function pipelineRunSucceeded(pipelineRun: Record | null): boolean | null { - if (pipelineRun === null) return null; - const conditions = Array.isArray(asOptionalRecord(pipelineRun.status)?.conditions) ? asOptionalRecord(pipelineRun.status)?.conditions : []; - for (const condition of conditions as unknown[]) { - const record = asOptionalRecord(condition); - if (record?.type !== "Succeeded") continue; - if (record.status === "True") return true; - if (record.status === "False") return false; - return null; - } - return null; -} - -function argoApplicationReady(application: Record | null): boolean { - if (application === null) return false; - const status = asOptionalRecord(application.status); - const sync = asOptionalRecord(status?.sync); - const health = asOptionalRecord(status?.health); - return sync?.status === "Synced" && health?.status === "Healthy"; -} - -function runtimeWorkloadsReady(runtime: NativeStatusSpec["runtime"], workloads: Record[]): boolean { - if (runtime === null) return true; - if (workloads.length < runtime.workloads.length) return false; - return runtime.workloads.every((spec, index) => workloadReady(spec, workloads[index] ?? null)); -} - -function workloadReady(spec: NativeWorkloadSpec, workload: Record | null): boolean { - if (workload === null) return false; - const status = asOptionalRecord(workload.status); - if (spec.kind === "Deployment") { - const desired = numberOrNull(asOptionalRecord(workload.spec)?.replicas) ?? 1; - const available = numberOrNull(status?.availableReplicas) ?? 0; - const updated = numberOrNull(status?.updatedReplicas) ?? 0; - return available >= desired && updated >= desired; - } - const desired = numberOrNull(asOptionalRecord(workload.spec)?.replicas) ?? 1; - const ready = numberOrNull(status?.readyReplicas) ?? 0; - return ready >= desired; -} - -function runtimeTargetShaFromWorkloads(runtime: NativeStatusSpec["runtime"], workloads: Record[]): string | null { - if (runtime === null) return null; - const commits: string[] = []; - runtime.workloads.forEach((spec, index) => { - const commit = workloadSourceCommit(spec, workloads[index] ?? null); - if (commit !== null) commits.push(commit); - }); - const unique = Array.from(new Set(commits)); - return unique.length === 1 ? unique[0] ?? null : null; -} - -function workloadSourceCommit(spec: NativeWorkloadSpec, workload: Record | null): string | null { - if (workload === null) return null; - const metadata = asOptionalRecord(workload.metadata); - const labels = asOptionalRecord(metadata?.labels); - const annotations = asOptionalRecord(metadata?.annotations); - const template = asOptionalRecord(asOptionalRecord(workload.spec)?.template); - const podMetadata = asOptionalRecord(template?.metadata); - const podLabels = asOptionalRecord(podMetadata?.labels); - const podAnnotations = asOptionalRecord(podMetadata?.annotations); - for (const key of spec.sourceCommit.labels) { - const value = shaOrNull(labels?.[key]); - if (value !== null) return value; - } - for (const key of spec.sourceCommit.annotations) { - const value = shaOrNull(annotations?.[key]); - if (value !== null) return value; - } - for (const key of spec.sourceCommit.podLabels) { - const value = shaOrNull(podLabels?.[key]); - if (value !== null) return value; - } - for (const key of spec.sourceCommit.podAnnotations) { - const value = shaOrNull(podAnnotations?.[key]); - if (value !== null) return value; - } - const containers = Array.isArray(asOptionalRecord(template?.spec)?.containers) ? asOptionalRecord(template?.spec)?.containers : []; - for (const envName of spec.sourceCommit.env) { - for (const container of containers as unknown[]) { - const env = Array.isArray(asOptionalRecord(container)?.env) ? asOptionalRecord(container)?.env : []; - for (const entry of env as unknown[]) { - const record = asOptionalRecord(entry); - if (record?.name !== envName) continue; - const value = shaOrNull(record.value); - if (value !== null) return value; - } - } - } - return null; -} - -function shaOrNull(value: unknown): string | null { - return typeof value === "string" && /^[0-9a-f]{40}$/iu.test(value) ? value : null; -} - -function nativePipelineRunSummary(pipelineRun: Record | null): Record | null { - if (pipelineRun === null) return null; - const metadata = asOptionalRecord(pipelineRun.metadata); - const status = asOptionalRecord(pipelineRun.status); - const condition = latestCondition(status, "Succeeded"); - return { - name: stringOrNull(metadata?.name), - namespace: stringOrNull(metadata?.namespace), - succeeded: pipelineRunSucceeded(pipelineRun), - reason: stringOrNull(condition?.reason), - startTime: stringOrNull(status?.startTime), - completionTime: stringOrNull(status?.completionTime), - durationSeconds: numberOrNull(status?.durationSeconds), - }; -} - -function nativeArgoSummary(application: Record | null): Record | null { - if (application === null) return null; - const metadata = asOptionalRecord(application.metadata); - const status = asOptionalRecord(application.status); - const sync = asOptionalRecord(status?.sync); - const health = asOptionalRecord(status?.health); - const operationState = asOptionalRecord(status?.operationState); - return { - name: stringOrNull(metadata?.name), - namespace: stringOrNull(metadata?.namespace), - syncStatus: stringOrNull(sync?.status), - healthStatus: stringOrNull(health?.status), - healthMessage: stringOrNull(health?.message), - revision: stringOrNull(sync?.revision), - operationPhase: stringOrNull(operationState?.phase), - operationMessage: stringOrNull(operationState?.message), - operationStartedAt: stringOrNull(operationState?.startedAt), - operationFinishedAt: stringOrNull(operationState?.finishedAt), - operationDurationSeconds: numberOrNull(operationState?.durationSeconds), - conditions: Array.isArray(status?.conditions) ? status.conditions.slice(0, 5) : [], - nonReadyResources: Array.isArray(status?.nonReadyResources) ? status.nonReadyResources.slice(0, 5) : [], - ready: argoApplicationReady(application), - }; -} - -function nativeRuntimeSummary(runtime: NativeStatusSpec["runtime"], workloads: Record[], expectedSha: string | null): Record | null { - if (runtime === null) return null; - const targetSha = runtimeTargetShaFromWorkloads(runtime, workloads); - return { - namespace: runtime.namespace, - ready: runtimeWorkloadsReady(runtime, workloads), - targetSha, - expectedSha, - aligned: expectedSha !== null && targetSha !== null ? targetSha === expectedSha : null, - workloads: runtime.workloads.map((spec, index) => { - const workload = workloads[index] ?? null; - const sourceCommit = workloadSourceCommit(spec, workload); - return { - kind: spec.kind, - name: spec.name, - ready: workloadReady(spec, workload), - sourceCommit, - aligned: expectedSha !== null && sourceCommit !== null ? sourceCommit === expectedSha : null, - }; - }), - }; -} - -function nativeGitMirrorGitopsBranch(follower: FollowerSpec): string | null { - if (follower.adapter === "hwlab-node-runtime") { - return hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node).gitopsBranch; - } - if (follower.adapter === "agentrun-yaml-lane") { - return resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }).spec.gitops.branch; - } - return null; -} - -function nativeGitMirrorRequired(follower: FollowerSpec): boolean { - return nativeGitMirrorGitopsBranch(follower) !== null - && (follower.closeoutChecks.includes("gitMirrorPostFlush") || follower.closeoutChecks.includes("gitops")); -} - -function nativeGitMirrorReady(gitMirror: Record | null): boolean { - if (gitMirror === null) return false; - if (gitMirror.ok === false) return false; - if (gitMirror.sourceSnapshotReady === false) return false; - if (gitMirror.pendingFlush === true) return false; - if (gitMirror.githubInSync === false) return false; - return true; -} - -function nativeGitMirrorSummary(gitMirror: Record | null): Record | null { - if (gitMirror === null) return null; - return { - ok: gitMirror.ok === true, - repository: stringOrNull(gitMirror.repository), - sourceBranch: stringOrNull(gitMirror.sourceBranch), - gitopsBranch: stringOrNull(gitMirror.gitopsBranch), - localSource: stringOrNull(gitMirror.localSource), - githubSource: stringOrNull(gitMirror.githubSource), - sourceStageRef: stringOrNull(gitMirror.sourceStageRef), - sourceSnapshotReady: gitMirror.sourceSnapshotReady === true, - localGitops: stringOrNull(gitMirror.localGitops), - githubGitops: stringOrNull(gitMirror.githubGitops), - pendingFlush: gitMirror.pendingFlush === true, - githubInSync: gitMirror.githubInSync === true, - statusAuthority: stringOrNull(gitMirror.statusAuthority) ?? "k8s-git-mirror-cache", - }; -} - -function latestCondition(status: Record | null, type: string): Record | null { - const conditions = Array.isArray(status?.conditions) ? status.conditions : []; - for (const condition of conditions as unknown[]) { - const record = asOptionalRecord(condition); - if (record?.type === type) return record; - } - return null; -} - -function mergeFollowerStatus( - registry: BranchFollowerRegistry, - follower: FollowerSpec, - stored: Record, - live: AdapterSummary | null, - liveRequested: boolean, -): Record { - const storedSource = asOptionalRecord(stored.source); - const storedTarget = asOptionalRecord(stored.target); - const phase = live?.phase ?? stringOrNull(stored.phase) ?? "Observed"; - const observedSha = live?.observedSha ?? stringOrNull(storedSource?.observedSha); - const targetSha = live?.targetSha ?? stringOrNull(storedTarget?.targetSha); - const lastTriggeredSha = live?.lastTriggeredSha ?? stringOrNull(stored.lastTriggeredSha); - const lastSucceededSha = live?.lastSucceededSha ?? stringOrNull(stored.lastSucceededSha); - return { - ok: live === null ? true : live.ok, - id: follower.id, - enabled: follower.enabled, - adapter: follower.adapter, - phase, - source: { - repository: follower.source.repository, - branch: follower.source.branch, - observedSha, - snapshotPrefix: follower.source.snapshotPrefix, - }, - target: { - node: follower.target.node, - lane: follower.target.lane, - namespace: follower.target.namespace, - sentinel: follower.target.sentinel, - targetSha, - }, - lastTriggeredSha, - lastSucceededSha, - pipelineRun: live?.pipelineRun ?? stringOrNull(stored.pipelineRun), - inFlightJob: live?.inFlightJob ?? stringOrNull(stored.inFlightJob), - budgetSource: follower.budgets, - updatedAt: stringOrNull(stored.updatedAt), - stateConfigMap: registry.controller.stateConfigMapName, - live: liveRequested, - message: live?.message ?? stringOrNull(stored.decision) ?? "no controller state yet", - timings: live === null ? storedFollowerTimingsForStatus(follower, asOptionalRecord(stored.timings), phase, observedSha) : buildFollowerTimings(follower, live, undefined, asOptionalRecord(stored.timings), phase), - warnings: Array.isArray(stored.warnings) ? stored.warnings.slice(0, 6) : [], - next: followerNextCommands(follower), - }; -} - -function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead { - const errors: string[] = []; - const stateResult = kubeConfigMapFollowerState(registry, options); - const namespace = registry.controller.namespace; - const deploymentResult = kubeJson(registry, options, `kubectl -n ${shQuote(namespace)} get deploy ${shQuote(registry.controller.deploymentName)} -o json`, 10_000, `/apis/apps/v1/namespaces/${encodeURIComponent(namespace)}/deployments/${encodeURIComponent(registry.controller.deploymentName)}`); - const leaseResult = kubeJson(registry, options, `kubectl -n ${shQuote(namespace)} get lease ${shQuote(registry.controller.leaseName)} -o json`, 10_000, `/apis/coordination.k8s.io/v1/namespaces/${encodeURIComponent(namespace)}/leases/${encodeURIComponent(registry.controller.leaseName)}`); - const podSelector = labelSelector(registry.controller.labels); - const podsResult = kubePodList(registry, options, podSelector); - if (!stateResult.ok) errors.push(`state configmap: ${stateResult.error}`); - if (!deploymentResult.ok && !isNotFoundText(deploymentResult.error)) errors.push(`deployment: ${deploymentResult.error}`); - if (!leaseResult.ok && !isNotFoundText(leaseResult.error)) errors.push(`lease: ${leaseResult.error}`); - if (!podsResult.ok && !isNotFoundText(podsResult.error)) errors.push(`pods: ${podsResult.error}`); - return { - ok: errors.length === 0, - stateByFollower: stateResult.stateByFollower, - stateMetadata: stateResult.metadata, - stateValueBytes: stateResult.valueBytes, - stateConfigMapPresent: stateResult.present, - deployment: deploymentResult.value, - lease: leaseResult.value, - pods: podsResult.value, - errors, - }; -} - -function kubeConfigMapFollowerState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sFollowerStateRead { - const followers = options.followerId === null ? registry.followers : registry.followers.filter((follower) => follower.id === options.followerId); - const maxTimingStages = options.followerId === null ? 8 : 16; - const script = [ - "set -eu", - "tmpdir=$(mktemp -d)", - "cleanup() { rm -rf \"$tmpdir\"; }", - "trap cleanup EXIT INT TERM", - nativeCicdScriptLoadShell(["read-state-summary.mjs"]), - `NAMESPACE=${shQuote(registry.controller.namespace)}`, - `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, - `FOLLOWERS_JSON=${shQuote(JSON.stringify(followers.map((follower) => follower.id)))}`, - `MAX_TIMING_STAGES=${maxTimingStages}`, - "export NAMESPACE CONFIGMAP FOLLOWERS_JSON MAX_TIMING_STAGES", - "node \"$tmpdir/read-state-summary.mjs\"", - ].join("\n"); - const result = runKubeScript(registry, options, script, "", 10_000); - const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; - if (parsed === null) { - const error = redactText(tailText(result.stderr || result.stdout, 800)); - return { ok: false, stateByFollower: {}, present: false, error }; - } - const parsedStates = asOptionalRecord(parsed.stateByFollower) ?? {}; - const metadata = asOptionalRecord(parsed.metadata); - const parsedValueBytes = asOptionalRecord(parsed.valueBytes) ?? {}; - const stateByFollower: Record> = {}; - const valueBytes: Record = {}; - for (const follower of followers) { - const state = asOptionalRecord(parsedStates[follower.id]); - if (state !== null) stateByFollower[follower.id] = state; - const bytes = numberOrNull(parsedValueBytes[follower.id]); - if (bytes !== null) valueBytes[follower.id] = bytes; - } - const errors = Array.isArray(parsed.errors) ? parsed.errors.map(String).filter((item) => item.length > 0) : []; - return { - ok: parsed.ok === true && errors.length === 0, - stateByFollower, - metadata, - valueBytes, - present: parsed.present === true, - error: errors.join("; "), - }; -} - -function stateWriteSummary(followerId: string, result: CommandResult): Record { - const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; - return { - follower: followerId, - ok: result.exitCode === 0 && parsed?.ok !== false, - exitCode: result.exitCode, - timedOut: result.timedOut, - beforeResourceVersion: stringOrNull(parsed?.beforeResourceVersion), - afterResourceVersion: stringOrNull(parsed?.afterResourceVersion), - preservedTiming: parsed?.preservedTiming === true, - message: result.exitCode === 0 ? "state patch command completed" : redactText(tailText(result.stderr || result.stdout, 500)), - parsedDownstreamCliOutput: false, - }; -} - -function removeFollowerStateKeys(registry: BranchFollowerRegistry, options: ParsedOptions, ids: string[]): CommandResult { - const patch = JSON.stringify({ data: Object.fromEntries(ids.map((id) => [id, null])) }); - const script = options.inCluster - ? [ - "set -eu", - "tmpdir=$(mktemp -d)", - "cleanup() { rm -rf \"$tmpdir\"; }", - "trap cleanup EXIT INT TERM", - nativeCicdScriptLoadShell(["patch-configmap-data.mjs"]), - `NAMESPACE=${shQuote(registry.controller.namespace)}`, - `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, - `PATCH_B64=${shQuote(Buffer.from(patch, "utf8").toString("base64"))}`, - "export NAMESPACE CONFIGMAP PATCH_B64", - "node \"$tmpdir/patch-configmap-data.mjs\"", - ].join("\n") - : [ - "set -eu", - "tmpdir=$(mktemp -d)", - "cleanup() { rm -rf \"$tmpdir\"; }", - "trap cleanup EXIT INT TERM", - `NAMESPACE=${shQuote(registry.controller.namespace)}`, - `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, - `PATCH=${shQuote(patch)}`, - "export NAMESPACE CONFIGMAP PATCH", - "if ! kubectl -n \"$NAMESPACE\" get configmap \"$CONFIGMAP\" >/dev/null 2>\"$tmpdir/error\"; then", - " if grep -qi 'not found' \"$tmpdir/error\"; then", - " printf '{\"ok\":true,\"present\":false,\"patched\":false,\"reason\":\"state-configmap-not-found\",\"parsedDownstreamCliOutput\":false}'", - " exit 0", - " fi", - " cat \"$tmpdir/error\" >&2", - " exit 1", - "fi", - "kubectl -n \"$NAMESPACE\" patch configmap \"$CONFIGMAP\" --type merge -p \"$PATCH\" >/dev/null", - "printf '{\"ok\":true,\"present\":true,\"patched\":true,\"parsedDownstreamCliOutput\":false}'", - ].join("\n"); - return runKubeScript(registry, options, script, "", 10_000); -} - -function kubeJson(registry: BranchFollowerRegistry, options: ParsedOptions, command: string, timeoutMs: number, inClusterPath?: string): { ok: boolean; value: Record | null; error: string } { - const script = options.inCluster && inClusterPath !== undefined - ? [ - "set -eu", - "tmpdir=$(mktemp -d)", - "cleanup() { rm -rf \"$tmpdir\"; }", - "trap cleanup EXIT INT TERM", - nativeCicdScriptLoadShell(["kube-get.mjs"]), - `node "$tmpdir/kube-get.mjs" ${shQuote(inClusterPath)}`, - ].join("\n") - : `set -eu\n${command}`; - const result = runKubeScript(registry, options, script, "", timeoutMs); - const value = result.exitCode === 0 ? parseJsonObject(result.stdout) : null; - return { - ok: result.exitCode === 0 && value !== null, - value, - error: redactText(tailText(result.stderr || result.stdout, 800)), - }; -} - -function kubePodList(registry: BranchFollowerRegistry, options: ParsedOptions, selector: string): { ok: boolean; value: Record | null; error: string } { - const command = `kubectl -n ${shQuote(registry.controller.namespace)} get pods -l ${shQuote(selector)} -o name`; - const script = options.inCluster - ? [ - "set -eu", - "tmpdir=$(mktemp -d)", - "cleanup() { rm -rf \"$tmpdir\"; }", - "trap cleanup EXIT INT TERM", - nativeCicdScriptLoadShell(["kube-get.mjs"]), - `node "$tmpdir/kube-get.mjs" ${shQuote(`/api/v1/namespaces/${encodeURIComponent(registry.controller.namespace)}/pods?labelSelector=${encodeURIComponent(selector)}`)}`, - ].join("\n") - : `set -eu\n${command}`; - const result = runKubeScript(registry, options, script, "", 10_000); - const parsed = options.inCluster && result.exitCode === 0 ? parseJsonObject(result.stdout) : null; - const names = options.inCluster - ? arrayRecords(parsed?.items).map((item) => stringOrNull(asOptionalRecord(item.metadata)?.name)).filter((name): name is string => name !== null) - : result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => line.replace(/^pod\//u, "")); - return { - ok: result.exitCode === 0, - value: result.exitCode === 0 ? { items: names.map((name) => ({ metadata: { name } })) } : null, - error: redactText(tailText(result.stderr || result.stdout, 800)), - }; -} - -function runKubeScript(registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number): CommandResult { - if (options.inCluster) { - return runCommand(["sh", "-lc", script], repoRoot, { input, timeoutMs }); - } - return runCommand([transPath(), registry.controller.kubeRoute, "sh"], repoRoot, { input: `${script}\n`, timeoutMs }); -} - -function compactFollowerStateForConfigMap(state: FollowerState): Record { - return { - id: state.id, - adapter: state.adapter, - enabled: state.enabled, - phase: state.phase, - source: state.source, - target: state.target, - lastTriggeredSha: state.lastTriggeredSha, - lastSucceededSha: state.lastSucceededSha, - pipelineRun: state.pipelineRun, - inFlightJob: state.inFlightJob, - budgetSource: state.budgetSource, - controller: state.controller, - decision: state.decision, - dryRun: state.dryRun, - updatedAt: state.updatedAt, - timings: compactTimings(state.timings), - warnings: compactStateWarnings(state.warnings), - stateFormat: "compact-v1", - command: compactStateCommand(state.command), - }; -} - -function compactStateCommand(command: Record | undefined): Record | undefined { - if (command === undefined) return undefined; - const closeout = asOptionalRecord(command.closeout); - const closeoutSummary = asOptionalRecord(closeout?.summary); - const payload = compactNativePayload(asOptionalRecord(command.payload)) ?? closeoutSummary; - return { - mode: stringOrNull(command.mode) ?? stringOrNull(command.status), - namespace: stringOrNull(command.namespace), - pipelineRun: stringOrNull(command.pipelineRun), - sourceCommit: stringOrNull(command.sourceCommit), - sourceStageRef: stringOrNull(command.sourceStageRef), - wait: command.wait === true ? true : undefined, - pipelineRunCompleted: command.pipelineRunCompleted === true ? true : undefined, - stillRunning: command.stillRunning === true ? true : undefined, - closeout: closeout === null - ? null - : { - ok: closeout.ok === true, - completed: closeout.completed === true, - timedOut: closeout.timedOut === true, - polls: numberOrNull(closeout.polls), - elapsedMs: numberOrNull(closeout.elapsedMs), - gitMirrorFlush: compactCloseoutGitMirrorFlush(asOptionalRecord(closeout.gitMirrorFlush)), - summary: closeoutSummary, - statusAuthority: stringOrNull(closeout.statusAuthority), - parsedDownstreamCliOutput: false, - }, - payload, - exitCode: numberOrNull(command.exitCode), - timedOut: command.timedOut === true, - statusAuthority: stringOrNull(command.statusAuthority), - parsedDownstreamCliOutput: false, - }; -} - -function compactCloseoutGitMirrorFlush(value: Record | null): Record | null { - if (value === null) return null; - const result = asOptionalRecord(value.result); - return { - jobName: stringOrNull(value.jobName), - namespace: stringOrNull(value.namespace), - result: result === null - ? null - : { - ok: result.ok === true, - completed: result.completed === true, - failed: result.failed === true, - timedOut: result.timedOut === true, - created: result.created === true, - reused: result.reused === true, - jobName: stringOrNull(result.jobName), - namespace: stringOrNull(result.namespace), - elapsedMs: numberOrNull(result.elapsedMs), - conditionReason: stringOrNull(result.conditionReason), - conditionMessage: stringOrNull(result.conditionMessage), - statusAuthority: stringOrNull(result.statusAuthority), - parsedDownstreamCliOutput: false, - }, - }; -} - -function compactStateWarnings(warnings: string[]): string[] { - return warnings.slice(0, 4).map((item) => redactText(item.length <= 800 ? item : `${item.slice(0, 400)} ... ${item.slice(-400)}`)); -} - -function compactNativePayload(payload: Record | null): Record | null { - if (payload === null) return null; - return { - source: compactSourcePayload(asOptionalRecord(payload.source)), - sourceSync: compactSourceSyncPayload(asOptionalRecord(payload.sourceSync)), - gitMirror: asOptionalRecord(payload.gitMirror), - tekton: asOptionalRecord(payload.tekton), - taskRuns: compactTaskRunsPayload(asOptionalRecord(payload.taskRuns)), - planArtifacts: compactPlanArtifactsPayload(asOptionalRecord(payload.planArtifacts)), - argo: asOptionalRecord(payload.argo), - runtime: asOptionalRecord(payload.runtime), - errors: Array.isArray(payload.errors) ? payload.errors.slice(0, 5) : [], - statusAuthority: stringOrNull(payload.statusAuthority), - parsedDownstreamCliOutput: false, - }; -} - -function compactSourceSyncPayload(value: Record | null): Record | null { - if (value === null) return null; - return { - ok: value.ok === true, - completed: value.completed === true, - failed: value.failed === true, - timedOut: value.timedOut === true, - created: value.created === true, - reused: value.reused === true, - jobName: stringOrNull(value.jobName), - namespace: stringOrNull(value.namespace), - elapsedMs: numberOrNull(value.elapsedMs), - conditionReason: stringOrNull(value.conditionReason), - conditionMessage: stringOrNull(value.conditionMessage), - statusAuthority: stringOrNull(value.statusAuthority), - parsedDownstreamCliOutput: false, - }; -} - -function compactTaskRunsPayload(taskRuns: Record | null): Record | null { - if (taskRuns === null) return null; - return { - ok: taskRuns.ok === true, - count: numberOrNull(taskRuns.count), - succeededCount: numberOrNull(taskRuns.succeededCount), - failedCount: numberOrNull(taskRuns.failedCount), - activeCount: numberOrNull(taskRuns.activeCount), - performance: asOptionalRecord(taskRuns.performance), - items: Array.isArray(taskRuns.items) ? taskRuns.items.slice(0, 16) : [], - }; -} - -function compactPlanArtifactsPayload(planArtifacts: Record | null): Record | null { - if (planArtifacts === null) return null; - return { - ok: planArtifacts.ok === true, - pipelineRun: stringOrNull(planArtifacts.pipelineRun), - eventFound: planArtifacts.eventFound === true, - degradedReason: stringOrNull(planArtifacts.degradedReason), - sourceCommitId: stringOrNull(planArtifacts.sourceCommitId), - affectedServices: Array.isArray(planArtifacts.affectedServices) ? planArtifacts.affectedServices.slice(0, 40) : [], - rolloutServices: Array.isArray(planArtifacts.rolloutServices) ? planArtifacts.rolloutServices.slice(0, 40) : [], - buildServices: Array.isArray(planArtifacts.buildServices) ? planArtifacts.buildServices.slice(0, 40) : [], - reusedServices: Array.isArray(planArtifacts.reusedServices) ? planArtifacts.reusedServices.slice(0, 40) : [], - buildSkippedCount: numberOrNull(planArtifacts.buildSkippedCount), - summary: stringOrNull(planArtifacts.summary), - disclosure: stringOrNull(planArtifacts.disclosure), - }; -} - -function compactSourcePayload(source: Record | null): Record | null { - if (source === null) return null; - return { - commit: stringOrNull(source.commit), - branch: stringOrNull(source.branch), - repository: stringOrNull(source.repository), - stageRef: stringOrNull(source.stageRef), - sourceAuthority: stringOrNull(source.sourceAuthority), - repoPath: stringOrNull(source.repoPath), - }; -} - -function buildFollowerTimings( - follower: FollowerSpec, - live: AdapterSummary, - triggerCommand: Record | undefined, - storedTimings?: Record | null, - phase?: BranchFollowerPhase, -): FollowerState["timings"] { - const nativePayload = asOptionalRecord(live.payload); - const finishOverride = stringOrNull(triggerCommand?.finishedAt) ?? noopStoredTotalFinishOverride(storedTimings, phase, live); - const total = totalTimingFromCommand(triggerCommand, phase) ?? totalTimingFromStored(storedTimings, phase, finishOverride, live.observedSha); - const stages = dedupeTimingStages([ - ...stageTimingsFromCommand(triggerCommand), - ...stageTimingsFromNativePayload(nativePayload), - ]).slice(0, 24); - const stageSourceCommit = stages.length > 0 ? live.observedSha : null; - return { - budgetSeconds: follower.budgets.endToEndSeconds, - totalSeconds: total?.seconds ?? null, - totalStatus: total?.status ?? "unknown", - totalSource: total?.source ?? "-", - sourceCommit: total?.sourceCommit ?? stringOrNull(triggerCommand?.sourceCommit) ?? stageSourceCommit, - startedAt: total?.startedAt ?? null, - finishedAt: total?.finishedAt ?? null, - overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, - stages, - }; -} - -function noopStoredTotalFinishOverride( - storedTimings: Record | null | undefined, - phase: BranchFollowerPhase | undefined, - live: AdapterSummary, -): string | null { - if (phase !== "Noop" || live.aligned !== true || live.observedSha === null) return null; - if (storedTimings === null || storedTimings === undefined) return null; - if (stringOrNull(storedTimings.sourceCommit) !== live.observedSha) return null; - if (stringOrNull(storedTimings.startedAt) === null) return null; - if (stringOrNull(storedTimings.finishedAt) !== null) return null; - return new Date().toISOString(); -} - -function storedFollowerTimingsForStatus( - follower: FollowerSpec, - storedTimings: Record | null, - phase: BranchFollowerPhase, - observedSha: string | null, -): FollowerState["timings"] { - const total = totalTimingFromStored(storedTimings, phase, null, observedSha); - const sourceCommit = total?.sourceCommit ?? stringOrNull(storedTimings?.sourceCommit) ?? null; - return { - budgetSeconds: follower.budgets.endToEndSeconds, - totalSeconds: total?.seconds ?? null, - totalStatus: total?.status ?? "unknown", - totalSource: total?.source ?? "-", - sourceCommit, - startedAt: total?.startedAt ?? null, - finishedAt: total?.finishedAt ?? null, - overBudget: total === null ? null : total.seconds > follower.budgets.endToEndSeconds, - stages: sourceCommit === null ? [] : storedStageTimings(storedTimings), - }; -} - -function storedStageTimings(storedTimings: Record | null): StageTiming[] { - if (storedTimings === null) return []; - return arrayRecords(storedTimings.stages) - .map((stage) => stageTiming( - stringOrNull(stage.stage) ?? "", - stringOrNull(stage.status) ?? "unknown", - numberOrNull(stage.seconds), - numberOrNull(stage.budgetSeconds), - stringOrNull(stage.source) ?? "stored-state", - stringOrNull(stage.object), - )) - .filter((stage) => stage.stage.length > 0) - .slice(0, 24); -} - -function compactTimings(timings: FollowerState["timings"]): FollowerState["timings"] { - return { - budgetSeconds: timings.budgetSeconds, - totalSeconds: timings.totalSeconds, - totalStatus: timings.totalStatus, - totalSource: timings.totalSource, - sourceCommit: timings.sourceCommit, - startedAt: timings.startedAt, - finishedAt: timings.finishedAt, - overBudget: timings.overBudget, - stages: timings.stages.slice(0, 24).map((stage) => ({ - stage: stage.stage, - status: stage.status, - seconds: stage.seconds, - budgetSeconds: stage.budgetSeconds, - source: stage.source, - object: stage.object, - })), - }; -} - -function totalTimingFromCommand(command: Record | undefined, phase?: BranchFollowerPhase): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null { - if (command === undefined) return null; - if (command.mode === "k8s-native-closeout") return null; - const payload = asOptionalRecord(command.payload); - if (payload?.reused === true) return null; - const startedAt = stringOrNull(command.startedAt); - const finishedAt = stringOrNull(command.finishedAt); - const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? secondsFromMs(numberOrNull(command.elapsedMs)); - if (seconds === null) return null; - const closeout = asOptionalRecord(command.closeout); - const exitCode = numberOrNull(command.exitCode); - const status = command.ok === false || (exitCode !== null && exitCode !== 0) - ? "failed" - : command.timedOut === true || closeout?.timedOut === true - ? "timed-out" - : closeout?.completed === true || command.completed === true - ? "completed" - : command.stillRunning === true - ? "running" - : command.pipelineRunCompleted === true - ? "ci-completed" - : phase === undefined - ? "submitted" - : phase.toLowerCase(); - return { seconds, status, source: stringOrNull(command.mode) ?? stringOrNull(command.status) ?? "command", sourceCommit: stringOrNull(command.sourceCommit), startedAt, finishedAt }; -} - -function totalTimingFromStored(storedTimings: Record | null | undefined, phase?: BranchFollowerPhase, finishOverride?: string | null, observedSha?: string | null): { seconds: number; status: string; source: string; sourceCommit: string | null; startedAt: string | null; finishedAt: string | null } | null { - if (storedTimings === null || storedTimings === undefined) return null; - const status = stringOrNull(storedTimings.totalStatus); - const source = stringOrNull(storedTimings.totalSource); - const sourceCommit = stringOrNull(storedTimings.sourceCommit); - if (sourceCommit === null) return null; - if (observedSha !== null && observedSha !== undefined && sourceCommit !== observedSha) return null; - const startedAt = stringOrNull(storedTimings.startedAt); - const finishedAt = stringOrNull(storedTimings.finishedAt) ?? finishOverride ?? null; - if (phase === "Noop" && finishedAt === null) return null; - const seconds = totalSecondsFromRange(startedAt, finishedAt) ?? numberOrNull(storedTimings.totalSeconds); - if (seconds === null) return null; - return { - seconds, - status: finishedAt === null && phase !== undefined && !terminalPhase(phase) ? phase.toLowerCase() : phase === undefined ? status ?? "recorded" : phase.toLowerCase(), - source: source ?? "stored-state", - sourceCommit, - startedAt, - finishedAt, - }; -} - -function totalSecondsFromRange(startedAt: string | null, finishedAt: string | null): number | null { - const startedMs = timestampMs(startedAt); - if (startedMs === null) return null; - const finishedMs = timestampMs(finishedAt) ?? Date.now(); - return finishedMs >= startedMs ? roundSeconds((finishedMs - startedMs) / 1000) : null; -} - -function timestampMs(value: string | null): number | null { - if (value === null) return null; - const parsed = Date.parse(value); - return Number.isFinite(parsed) ? parsed : null; -} - -function terminalPhase(phase: BranchFollowerPhase): boolean { - return phase === "Succeeded" || phase === "Failed" || phase === "Blocked" || phase === "Skipped" || phase === "Noop"; -} - -function stageTimingsFromNativePayload(payload: Record | null): StageTiming[] { - if (payload === null) return []; - const stages: StageTiming[] = []; - const statusRead = asOptionalRecord(asOptionalRecord(payload.timings)?.statusRead); - stages.push(stageTiming("status-read", "ok", secondsFromMs(numberOrNull(statusRead?.elapsedMs)), numberOrNull(statusRead?.budgetSeconds), "native-status", null)); - const sourceSyncStage = k8sJobTiming("git-mirror-sync", asOptionalRecord(payload.sourceSync)); - if (sourceSyncStage !== null) stages.push(sourceSyncStage); - const gitMirror = asOptionalRecord(payload.gitMirror); - if (gitMirror !== null) { - const hasGitopsBranch = stringOrNull(gitMirror.gitopsBranch) !== null; - const sourceReady = gitMirror.sourceSnapshotReady === true; - const status = gitMirror.pendingFlush === true - ? "pending-flush" - : hasGitopsBranch - ? gitMirror.githubInSync === true && sourceReady ? "ready" : "not-ready" - : sourceReady ? "source-ready" : "source-not-ready"; - stages.push(stageTiming("git-mirror", status, null, null, "git-mirror-cache", stringOrNull(gitMirror.gitopsBranch) ?? stringOrNull(gitMirror.sourceBranch))); - } - const tekton = asOptionalRecord(payload.tekton); - if (tekton !== null) { - const status = tekton.succeeded === true ? "succeeded" : tekton.succeeded === false ? `failed:${stringOrNull(tekton.reason) ?? "unknown"}` : "running"; - stages.push(stageTiming("pipelinerun", status, numberOrNull(tekton.durationSeconds), null, "tekton", stringOrNull(tekton.name))); - } - const taskRuns = asOptionalRecord(payload.taskRuns); - const taskRunItems = taskRuns !== null && Array.isArray(taskRuns.items) ? taskRuns.items : []; - for (const item of taskRunItems) { - const record = asOptionalRecord(item); - if (record === null) continue; - const name = stringOrNull(record.pipelineTask) ?? stringOrNull(record.name) ?? "unknown"; - const status = record.status === "True" ? "succeeded" : record.status === "False" ? `failed:${stringOrNull(record.reason) ?? "unknown"}` : "running"; - stages.push(stageTiming(`task:${name}`, status, numberOrNull(record.durationSeconds), null, "tekton-taskrun", stringOrNull(record.name))); - } - const argo = asOptionalRecord(payload.argo); - if (argo !== null) { - stages.push(stageTiming("argo", `${stringOrNull(argo.syncStatus) ?? "unknown"}/${stringOrNull(argo.healthStatus) ?? "unknown"}`, numberOrNull(argo.operationDurationSeconds), null, "argocd", stringOrNull(argo.name))); - } - const runtime = asOptionalRecord(payload.runtime); - if (runtime !== null) { - const aligned = runtime.aligned === true ? "aligned" : runtime.aligned === false ? "stale" : "unknown-target"; - stages.push(stageTiming("runtime", `${runtime.ready === true ? "ready" : "not-ready"}/${aligned}`, null, null, "kubernetes-workload", stringOrNull(runtime.namespace))); - } - return stages; -} - -function stageTimingsFromCommand(command: Record | undefined): StageTiming[] { - if (command === undefined) return []; - const stages: StageTiming[] = []; - const phase = stringOrNull(command.phase); - const jobStage = phase === null ? null : k8sJobTiming(phase, asOptionalRecord(command.job), stringOrNull(command.jobName)); - if (jobStage !== null) stages.push(jobStage); - const payload = asOptionalRecord(command.payload); - if (payload !== null) { - const capabilities = asOptionalRecord(payload.nativeCapabilities); - for (const stage of [ - k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)), - k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)), - k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)), - ]) { - if (stage !== null) stages.push(stage); - } - const agentrun = asOptionalRecord(payload.agentrun); - const agentrunSync = asOptionalRecord(agentrun?.gitMirrorSync); - const agentrunFlush = asOptionalRecord(agentrun?.gitMirrorFlush); - const imageBuild = asOptionalRecord(agentrun?.imageBuild); - const gitopsPublish = asOptionalRecord(agentrun?.gitopsPublish); - for (const stage of [ - k8sJobTiming("git-mirror-sync", asOptionalRecord(agentrunSync?.payload), stringOrNull(agentrunSync?.jobName)), - k8sJobTiming("image-build", asOptionalRecord(imageBuild?.result), stringOrNull(imageBuild?.jobName)), - k8sJobTiming("gitops-publish", asOptionalRecord(gitopsPublish?.result), stringOrNull(gitopsPublish?.jobName)), - k8sJobTiming("git-mirror-flush", asOptionalRecord(agentrunFlush?.payload), stringOrNull(agentrunFlush?.jobName)), - ]) { - if (stage !== null) stages.push(stage); - } - const tektonSeconds = secondsFromMs(numberOrNull(payload.elapsedMs)); - if (tektonSeconds !== null) { - const status = payload.completed === true ? "completed" : payload.failed === true ? "failed" : payload.stillRunning === true ? "running" : "submitted"; - stages.push(stageTiming("pipelinerun-wait", status, tektonSeconds, null, "tekton-submit", stringOrNull(command.pipelineRun))); - } - } - const closeout = asOptionalRecord(command.closeout); - if (closeout !== null) { - const gitMirrorFlush = asOptionalRecord(closeout.gitMirrorFlush); - const gitMirrorFlushStage = k8sJobTiming("git-mirror-flush", asOptionalRecord(gitMirrorFlush?.result), stringOrNull(gitMirrorFlush?.jobName)); - if (gitMirrorFlushStage !== null) stages.push(gitMirrorFlushStage); - const status = closeout.completed === true ? "completed" : closeout.timedOut === true ? "timed-out" : "pending"; - stages.push(stageTiming("closeout", status, secondsFromMs(numberOrNull(closeout.elapsedMs)), null, "k8s-native-closeout", stringOrNull(command.pipelineRun))); - } - return stages; -} - -function k8sJobTiming(stage: string, job: Record | null, objectOverride?: string | null): StageTiming | null { - if (job === null) return null; - const status = job.completed === true - ? job.reused === true ? "reused" : "completed" - : job.failed === true - ? "failed" - : job.timedOut === true - ? "timed-out" - : "running"; - return stageTiming(stage, status, secondsFromMs(numberOrNull(job.elapsedMs)), null, "kubernetes-job", objectOverride ?? stringOrNull(job.jobName)); -} - -function stageTiming(stage: string, status: string, seconds: number | null, budgetSeconds: number | null, source: string, object: string | null): StageTiming { - return { stage, status, seconds, budgetSeconds, source, object }; -} - -function dedupeTimingStages(stages: StageTiming[]): StageTiming[] { - const byKey = new Map(); - for (const stage of stages) { - if (stage.stage.length === 0) continue; - const key = `${stage.stage}\t${stage.object ?? ""}`; - const previous = byKey.get(key); - if (previous === undefined || previous.seconds === null && stage.seconds !== null) byKey.set(key, stage); - } - return [...byKey.values()]; -} - -function secondsFromMs(value: number | null): number | null { - return value === null ? null : roundSeconds(value / 1000); -} - -function roundSeconds(value: number): number { - return Math.round(value * 10) / 10; -} - -function writeFollowerState(registry: BranchFollowerRegistry, state: FollowerState, options: ParsedOptions): CommandResult { - const stateJson = JSON.stringify(compactFollowerStateForConfigMap(state)); - const script = [ - "set -eu", - `NAMESPACE=${shQuote(registry.controller.namespace)}`, - `CONFIGMAP=${shQuote(registry.controller.stateConfigMapName)}`, - `FOLLOWER_ID=${shQuote(state.id)}`, - `SPEC_REF=${shQuote(SPEC_REF)}`, - `STATE_B64=${shQuote(Buffer.from(stateJson, "utf8").toString("base64"))}`, - "export NAMESPACE CONFIGMAP FOLLOWER_ID SPEC_REF STATE_B64", - "tmpdir=$(mktemp -d)", - "cleanup() { rm -rf \"$tmpdir\"; }", - "trap cleanup EXIT INT TERM", - nativeCicdScriptLoadShell(["patch-follower-state.mjs"]), - "node \"$tmpdir/patch-follower-state.mjs\"", - ].join("\n"); - return runKubeScript(registry, options, script, "", 10_000); -} - -function runControllerReconcileJob(registry: BranchFollowerRegistry, options: ParsedOptions, mode: { dryRun: boolean; wait: boolean; recordState: boolean }): Record { - const timeoutSeconds = options.timeoutSeconds ?? registry.controller.budgets.runOnceSeconds; - const jobName = `${registry.controller.deploymentName}-once-${Date.now().toString(36)}`.slice(0, 63); - const manifest = renderControllerReconcileJob(registry, options, jobName, mode, timeoutSeconds); - const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`; - const manifestBase64 = Buffer.from(manifestYaml, "utf8").toString("base64"); - const script = [ - "set -eu", - "tmp=$(mktemp)", - "base64 -d >\"$tmp\" <<'UNIDESK_CICD_RECONCILE_JOB_B64'", - manifestBase64, - "UNIDESK_CICD_RECONCILE_JOB_B64", - `kubectl -n ${shQuote(registry.controller.namespace)} delete job ${shQuote(jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`, - `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`, - mode.wait ? waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds) : "true", - ].join("\n"); - const result = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000); - return { - ok: result.exitCode === 0, - name: jobName, - namespace: registry.controller.namespace, - mode: mode.dryRun ? "status-refresh" : "confirm-reconcile", - execution: "k8s-native-job", - exitCode: result.exitCode, - timedOut: result.timedOut, - message: result.exitCode === 0 ? "reconcile job completed" : redactText(tailText(result.stderr || result.stdout, 800)), - stdoutTail: redactText(tailText(result.stdout, options.full ? 4000 : 1000)), - stderrTail: redactText(tailText(result.stderr, options.full ? 2000 : 800)), - }; -} - -function selectFollowers(registry: BranchFollowerRegistry, options: ParsedOptions, opts: { includeDisabled: boolean }): FollowerSpec[] { - let selected = registry.followers; - if (options.followerId !== null) selected = selected.filter((item) => item.id === options.followerId); - else if (!options.all && options.action === "run-once") selected = selected.filter((item) => item.enabled); - if (!opts.includeDisabled) selected = selected.filter((item) => item.enabled); - if (selected.length === 0) throw new Error(options.followerId === null ? "no followers selected" : `unknown or disabled follower ${options.followerId}`); - return selected; -} - -function registrySummary(registry: BranchFollowerRegistry): Record { - return { - path: registry.path, - sha256: registry.rawSha256, - metadata: registry.metadata, - controller: { - namespace: registry.controller.namespace, - kubeRoute: registry.controller.kubeRoute, - deploymentName: registry.controller.deploymentName, - stateConfigMapName: registry.controller.stateConfigMapName, - leaseName: registry.controller.leaseName, - }, - followers: registry.followers.map((item) => item.id), - }; -} - -function redactCommands(follower: FollowerSpec): Record { - return { - plan: follower.commands.plan.argv.join(" "), - status: "native:k8s-git-mirror+tekton+argocd+runtime", - trigger: follower.commands.trigger.argv.join(" "), - events: follower.commands.events.argv.join(" "), - logs: follower.commands.logs.argv.join(" "), - }; -} - -function nativeStatusPlan(native: NativeStatusSpec): Record { - return { - source: { - mode: "k8s-git-mirror-cache", - gitMirrorNamespace: native.source.gitMirrorNamespace, - gitMirrorDeployment: native.source.gitMirrorDeployment, - repoPath: native.source.repoPath, - }, - tekton: native.tekton, - argo: native.argo, - runtime: native.runtime === null - ? null - : { - namespace: native.runtime.namespace, - workloads: native.runtime.workloads.map((item) => ({ kind: item.kind, name: item.name })), - }, - parsedDownstreamCliOutput: false, - }; -} - -function controllerStatusSummary(registry: BranchFollowerRegistry, k8s: K8sStateRead): Record { - const deploymentStatus = asOptionalRecord(k8s.deployment?.status); - const available = numberOrNull(deploymentStatus?.availableReplicas) ?? 0; - const replicas = numberOrNull(deploymentStatus?.replicas) ?? 0; - const leaseSpec = asOptionalRecord(k8s.lease?.spec); - const podItems = Array.isArray(k8s.pods?.items) ? k8s.pods.items.length : null; - return { - namespace: registry.controller.namespace, - route: registry.controller.kubeRoute, - deploymentName: registry.controller.deploymentName, - deploymentPresent: k8s.deployment !== null, - availableReplicas: available, - replicas, - pods: podItems, - stateConfigMapName: registry.controller.stateConfigMapName, - stateConfigMapPresent: k8s.stateConfigMapPresent, - leaseName: registry.controller.leaseName, - leaseHolder: stringOrNull(leaseSpec?.holderIdentity), - noHostWorktreeAuthority: true, - }; -} - -function followerNextCommands(follower: FollowerSpec): Record { - const next: Record = { - status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`, - liveStatus: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --live`, - dryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`, - trigger: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --confirm --wait`, - events: `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`, - logs: `bun scripts/cli.ts cicd branch-follower logs --follower ${follower.id}`, - }; - if (follower.nativeStatus.tekton !== null) next.pipelineRuns = `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`; - if (follower.nativeStatus.argo !== null) next.argoApplication = `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`; - return next; -} - -function safeResolveString(ref: string): string | null { - try { - return resolveConfigRefString(ref, ref); - } catch { - return null; - } -} - -function parseJsonObject(text: string): Record | null { - const trimmed = text.trim(); - if (trimmed.length === 0) return null; - try { - const parsed = JSON.parse(trimmed) as unknown; - return asOptionalRecord(parsed); - } catch { - const start = trimmed.indexOf("{"); - const end = trimmed.lastIndexOf("}"); - if (start < 0 || end <= start) return null; - try { - const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown; - return asOptionalRecord(parsed); - } catch { - return null; - } - } -} - -function recordAt(root: Record, path: string[]): Record | null { - let current: unknown = root; - for (const item of path) { - if (typeof current !== "object" || current === null || Array.isArray(current)) return null; - current = (current as Record)[item]; - } - return asOptionalRecord(current); -} - -function asOptionalRecord(value: unknown): Record | null { - if (typeof value !== "object" || value === null || Array.isArray(value)) return null; - return value as Record; -} - -function stringOrNull(value: unknown): string | null { - return typeof value === "string" && value.length > 0 ? value : null; -} - -function numberOrNull(value: unknown): number | null { - return typeof value === "number" && Number.isFinite(value) ? value : null; -} - -function commandCompact(result: CommandResult, options: ParsedOptions): Record { - return { - argv: result.command, - exitCode: result.exitCode, - timedOut: result.timedOut, - stdoutBytes: Buffer.byteLength(result.stdout), - stderrBytes: Buffer.byteLength(result.stderr), - stdoutTail: options.full || result.exitCode !== 0 ? redactText(tailText(result.stdout, options.tailBytes)) : "", - stderrTail: options.full || result.exitCode !== 0 ? redactText(tailText(result.stderr, Math.min(options.tailBytes, 4000))) : "", - }; -} - -function objectRef(item: Record): Record { - const metadata = asRecord(item.metadata, "metadata"); - return { - kind: stringField(item, "kind", "manifest"), - namespace: typeof metadata.namespace === "string" ? metadata.namespace : "-", - name: stringField(metadata, "name", "manifest.metadata"), - }; -} - -function labelSelector(labels: Record): string { - return Object.entries(labels).map(([key, value]) => `${key}=${value}`).join(","); -} - -function isNotFoundText(value: string): boolean { - return /notfound|not found|notfound|NotFound/u.test(value); -} - -function shortSha(value: string | null): string { - if (value === null) return "-"; - return value.length > 12 ? value.slice(0, 12) : value; -} - -function safeJobSegment(value: string): string { - return value.replace(/[^A-Za-z0-9_.-]/gu, "_").slice(0, 60); -} - -function nativeCapabilityJobName(followerId: string, action: string, sha: string): string { - const prefix = `${safeK8sNameSegment(followerId)}-${safeK8sNameSegment(action)}`; - return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63); -} - -function safeK8sNameSegment(value: string): string { - const normalized = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, ""); - return (normalized.length === 0 ? "x" : normalized).slice(0, 40).replace(/-$/u, ""); -} - -function tailText(text: string, maxChars: number): string { - if (text.length <= maxChars) return text; - return text.slice(text.length - maxChars); -} - -function commandLabel(options: ParsedOptions): string { - return `cicd branch-follower ${options.action}`; -} - -function arrayRecords(value: unknown): Record[] { - return Array.isArray(value) ? value.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; +export async function runCicdCommand(config: UniDeskConfig | null, args: string[]): Promise { + return await runBranchFollowerCommand(config, args); }