// SPEC: PJ2026-01060703 CI/CD branch follower independently executable gate probes. // Responsibility: submit bounded target-side gate Jobs and return compact evidence. import type { CommandResult } from "./command"; import { resolveAgentRunLaneTarget } from "./agentrun-lanes"; import { yamlLaneGitMirrorJobManifest } from "./agentrun/secrets"; import { nativeHwlabControlPlaneRefreshJobManifest, runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh"; import { nativeCicdScriptLoadShell } from "./cicd-native-bundle"; import { runNativeK8sJob } from "./cicd-native"; import { waitForJobShell } from "./cicd-controller-render"; import type { BranchFollowerRegistry, FollowerSpec, ParsedOptions } from "./cicd-types"; import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; import { nodeRuntimeGitMirrorJobManifest } from "./hwlab-node/render"; import { nodeRuntimeGitMirrorTarget } from "./hwlab-node/web-probe"; import { shQuote, redactText } from "./platform-infra-ops-library"; type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult; export async function runBranchFollowerGate(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner): Promise> { if (options.gate === null) throw new Error("gate requires --gate "); if (options.gate === "control-plane-refresh") { return options.inCluster ? runControlPlaneRefreshGate(registry, follower, options) : runTargetControlPlaneRefreshGateJob(registry, follower, options, runKubeScript); } if (options.gate === "git-mirror-flush") { return options.inCluster ? runGitMirrorFlushGate(registry, follower, options) : runTargetGitMirrorFlushGateJob(registry, follower, options, runKubeScript); } if (options.gate === "runtime-closeout" && !options.confirm) { const timeoutSeconds = gateTimeoutSeconds(follower, options); const jobName = `bf-gate-${safeName(follower.id)}-${safeName(options.gate)}-${Date.now().toString(36)}`.slice(0, 63); return { ok: true, action: "gate", gate: options.gate, follower: follower.id, dryRun: true, sourceCommit: options.sourceCommit, target: { name: jobName, namespace: registry.controller.namespace, execution: "k8s-native-gate-job" }, timeoutSeconds, message: "add --confirm to run the native runtime closeout gate", writesState: false, parsedDownstreamCliOutput: false, }; } if (options.inCluster) return { ok: false, action: "gate", gate: options.gate, follower: follower.id, degradedReason: "operator-entry-required" }; const timeoutSeconds = gateTimeoutSeconds(follower, options); const jobName = `bf-gate-${safeName(follower.id)}-${safeName(options.gate)}-${Date.now().toString(36)}`.slice(0, 63); const manifest = gateJobManifest(registry, follower, options, jobName, timeoutSeconds); const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`; const script = [ "set -eu", "tmp=$(mktemp)", "base64 -d >\"$tmp\" <<'UNIDESK_GATE_JOB'", Buffer.from(manifestYaml, "utf8").toString("base64"), "UNIDESK_GATE_JOB", `kubectl -n ${shQuote(registry.controller.namespace)} delete job ${shQuote(jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`, `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`, waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds), ].join("\n"); const startedAt = Date.now(); const command = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000); const parsed = command.exitCode === 0 ? parseLastJsonObject(command.stdout) : null; const ok = command.exitCode === 0 && parsed !== null && parsed.ok === true; return { ok, action: "gate", gate: options.gate, follower: follower.id, target: { name: jobName, namespace: registry.controller.namespace, execution: "k8s-native-gate-job" }, result: parsed, command: { exitCode: command.exitCode, timedOut: command.timedOut, elapsedMs: Date.now() - startedAt, parseError: parsed === null ? "stdout-json-parse-failed" : null, stdoutTail: parsed === null ? redactText(tailText(command.stdout, 1600)) : "", stderrTail: ok ? "" : redactText(tailText(command.stderr, 1200)), }, parsedDownstreamCliOutput: false, }; } function gateTimeoutSeconds(follower: FollowerSpec, options: ParsedOptions): number { return options.timeoutSeconds ?? (options.gate === "runtime-closeout" ? follower.budgets.endToEndSeconds : follower.budgets.statusSeconds); } function runTargetGitMirrorFlushGateJob(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner): Record { const prepared = prepareGitMirrorFlushGate(follower, options); if (prepared.ok !== true) return prepared; const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.sourceSyncSeconds; if (!options.confirm) return gitMirrorFlushDryRun(follower, options, prepared.namespace, prepared.jobName); const manifestYaml = `${Bun.YAML.stringify(prepared.manifest).trim()}\n`; const script = [ "set -eu", "tmp=$(mktemp)", "base64 -d >\"$tmp\" <<'UNIDESK_GIT_MIRROR_FLUSH_GATE_JOB'", Buffer.from(manifestYaml, "utf8").toString("base64"), "UNIDESK_GIT_MIRROR_FLUSH_GATE_JOB", `kubectl -n ${shQuote(prepared.namespace)} delete job ${shQuote(prepared.jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`, `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`, waitForJobShell(prepared.namespace, prepared.jobName, timeoutSeconds), ].join("\n"); const startedAt = Date.now(); const command = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000); const parsed = parseLastJsonObject(command.stdout); const ok = command.exitCode === 0 && parsed !== null && parsed.pendingFlush !== true; return { ok, action: "gate", gate: options.gate, follower: follower.id, dryRun: false, sourceCommit: options.sourceCommit, target: { name: prepared.jobName, namespace: prepared.namespace, execution: "k8s-native-git-mirror-flush" }, result: parsed, writesState: false, command: { exitCode: command.exitCode, timedOut: command.timedOut, elapsedMs: Date.now() - startedAt, parseError: parsed === null ? "stdout-json-parse-failed" : null, stdoutTail: ok ? "" : redactText(tailText(command.stdout, 1600)), stderrTail: ok ? "" : redactText(tailText(command.stderr, 1200)), }, parsedDownstreamCliOutput: false, next: { statusRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${follower.id} --step status-read --json`, job: `bun scripts/cli.ts cicd branch-follower job --follower ${follower.id} --source-commit ${options.sourceCommit} --job git-mirror-flush --json`, }, }; } function runGitMirrorFlushGate(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Record { const prepared = prepareGitMirrorFlushGate(follower, options); if (prepared.ok !== true) return prepared; const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.sourceSyncSeconds; if (!options.confirm) return gitMirrorFlushDryRun(follower, options, prepared.namespace, prepared.jobName); const startedAt = Date.now(); const result = runNativeK8sJob(prepared.namespace, prepared.jobName, prepared.manifest, timeoutSeconds, "flush", registry.controller.budgets); const summary = result.summary; const ok = result.ok && summary?.pendingFlush !== true; return { ok, action: "gate", gate: options.gate, follower: follower.id, dryRun: false, sourceCommit: options.sourceCommit, target: { name: prepared.jobName, namespace: prepared.namespace, execution: "k8s-native-git-mirror-flush" }, result: { ok: result.ok, completed: result.completed, failed: result.failed, timedOut: result.timedOut, created: result.created, reused: result.reused, polls: result.polls, elapsedMs: result.elapsedMs, summary, conditionReason: result.conditionReason, conditionMessage: result.conditionMessage, statusAuthority: result.statusAuthority, parsedDownstreamCliOutput: false, }, writesState: false, command: { elapsedMs: Date.now() - startedAt, timeoutSeconds, }, parsedDownstreamCliOutput: false, next: { statusRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${follower.id} --step status-read --json`, job: `bun scripts/cli.ts cicd branch-follower job --follower ${follower.id} --source-commit ${options.sourceCommit} --job git-mirror-flush --json`, }, }; } function prepareGitMirrorFlushGate(follower: FollowerSpec, options: ParsedOptions): { ok: true; namespace: string; jobName: string; manifest: Record } | Record { if (options.sourceCommit === null) { return { ok: false, action: "gate", gate: options.gate, follower: follower.id, degradedReason: "source-commit-required", message: "git-mirror-flush gate requires --source-commit ", parsedDownstreamCliOutput: false, }; } const jobName = nativeCapabilityJobName(follower.id, "flush", options.sourceCommit); if (follower.adapter === "hwlab-node-runtime") { const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); const mirror = nodeRuntimeGitMirrorTarget(spec); return { ok: true, namespace: mirror.namespace, jobName, manifest: nodeRuntimeGitMirrorJobManifest(mirror, "flush", jobName) }; } if (follower.adapter === "agentrun-yaml-lane") { const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }); return { ok: true, namespace: spec.gitMirror.namespace, jobName, manifest: yamlLaneGitMirrorJobManifest(spec, "flush", jobName) }; } return { ok: false, action: "gate", gate: options.gate, follower: follower.id, degradedReason: "unsupported-follower-adapter", message: "git-mirror-flush gate is only available for followers with a native git-mirror stage", parsedDownstreamCliOutput: false, }; } function gitMirrorFlushDryRun(follower: FollowerSpec, options: ParsedOptions, namespace: string, jobName: string): Record { return { ok: true, action: "gate", gate: options.gate, follower: follower.id, dryRun: true, sourceCommit: options.sourceCommit, target: { name: jobName, namespace, execution: "k8s-native-git-mirror-flush" }, message: "add --confirm to run the native git-mirror flush gate", writesState: false, parsedDownstreamCliOutput: false, }; } function runTargetControlPlaneRefreshGateJob(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner): Record { if (follower.adapter !== "hwlab-node-runtime" || options.sourceCommit === null || !options.confirm) { return runControlPlaneRefreshGate(registry, follower, options); } const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.controlPlaneRefreshSeconds; const jobName = nativeCapabilityJobName(follower.id, "control-plane-refresh", options.sourceCommit); const manifest = nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, options.sourceCommit, jobName, timeoutSeconds); const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`; const script = [ "set -eu", "tmp=$(mktemp)", "base64 -d >\"$tmp\" <<'UNIDESK_CONTROL_PLANE_REFRESH_GATE_JOB'", Buffer.from(manifestYaml, "utf8").toString("base64"), "UNIDESK_CONTROL_PLANE_REFRESH_GATE_JOB", `kubectl -n ${shQuote(registry.controller.namespace)} delete job ${shQuote(jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`, `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`, waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds), ].join("\n"); const startedAt = Date.now(); const command = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000); const parsed = command.exitCode === 0 ? parseLastJsonObject(command.stdout, isControlPlaneRefreshResult) : null; const ok = command.exitCode === 0 && parsed !== null && parsed.ok === true; return { ok, action: "gate", gate: options.gate, follower: follower.id, target: { name: jobName, namespace: registry.controller.namespace, execution: "k8s-native-control-plane-refresh" }, result: parsed, command: { exitCode: command.exitCode, timedOut: command.timedOut, elapsedMs: Date.now() - startedAt, parseError: parsed === null ? "stdout-json-parse-failed" : null, stdoutTail: ok ? "" : redactText(tailText(command.stdout, 1600)), stderrTail: ok ? "" : redactText(tailText(command.stderr, 1200)), }, parsedDownstreamCliOutput: false, }; } function runControlPlaneRefreshGate(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Record { if (follower.adapter !== "hwlab-node-runtime") { return { ok: false, action: "gate", gate: options.gate, follower: follower.id, degradedReason: "unsupported-follower-adapter", message: "control-plane-refresh gate is only available for hwlab-node-runtime followers", parsedDownstreamCliOutput: false, }; } if (options.sourceCommit === null) { return { ok: false, action: "gate", gate: options.gate, follower: follower.id, degradedReason: "source-commit-required", message: "control-plane-refresh gate requires --source-commit ", parsedDownstreamCliOutput: false, }; } const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.controlPlaneRefreshSeconds; const jobName = nativeCapabilityJobName(follower.id, "control-plane-refresh", options.sourceCommit); if (!options.confirm) { return { ok: true, action: "gate", gate: options.gate, follower: follower.id, dryRun: true, target: { name: jobName, namespace: registry.controller.namespace, execution: "k8s-native-control-plane-refresh" }, sourceCommit: options.sourceCommit, message: "add --confirm to run the native control-plane refresh gate", parsedDownstreamCliOutput: false, }; } const startedAt = Date.now(); const refresh = runNativeHwlabControlPlaneRefresh(registry, follower, spec, options.sourceCommit, timeoutSeconds, jobName); return { ok: refresh.result.ok, action: "gate", gate: options.gate, follower: follower.id, dryRun: false, sourceCommit: options.sourceCommit, target: { name: refresh.jobName, namespace: refresh.namespace, execution: "k8s-native-control-plane-refresh" }, result: refresh.result, command: { elapsedMs: Date.now() - startedAt, timeoutSeconds, }, parsedDownstreamCliOutput: false, next: { statusRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${follower.id} --step status-read --json`, job: `bun scripts/cli.ts cicd branch-follower job --follower ${follower.id} --source-commit ${options.sourceCommit} --job control-plane-refresh --json`, }, }; } function gateJobManifest(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, jobName: string, timeoutSeconds: number): Record { const labels = { ...registry.controller.labels, "app.kubernetes.io/component": "cicd-gate-job" }; const agentrun = follower.adapter === "agentrun-yaml-lane" ? resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }).spec : null; const hwlab = follower.adapter === "hwlab-node-runtime" ? hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node) : null; const gitopsBranch = agentrun?.gitops.branch ?? hwlab?.gitopsBranch ?? ""; const healthUrl = gateHealthUrl(follower); const workloads = (follower.nativeStatus.runtime?.workloads ?? []).map((item) => ({ kind: item.kind, name: item.name, sourceCommit: item.sourceCommit })); const gatePolicy = gatePolicyEnv(follower); const gateScript = [ "set -eu", "tmpdir=$(mktemp -d)", "cleanup() { rm -rf \"$tmpdir\"; }", "trap cleanup EXIT INT TERM", nativeCicdScriptLoadShell(["branch-follower-gate.sh", "branch-follower-gate.mjs", "reuse-config-summary.mjs", "plan-artifacts.mjs"]), "\"$tmpdir/branch-follower-gate.sh\"", ].join("\n"); return { apiVersion: "batch/v1", kind: "Job", metadata: { name: jobName, namespace: registry.controller.namespace, labels }, spec: { backoffLimit: registry.controller.budgets.reconcileJobBackoffLimit, ttlSecondsAfterFinished: registry.controller.budgets.reconcileJobTtlSeconds, activeDeadlineSeconds: timeoutSeconds + registry.controller.budgets.reconcileJobDeadlineGraceSeconds, template: { metadata: { labels }, spec: { restartPolicy: "Never", serviceAccountName: registry.controller.serviceAccountName, volumes: [ { name: "registry", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } }, { name: "git-mirror-cache", persistentVolumeClaim: { claimName: registry.controller.source.gitMirrorCachePvcName } }, { name: "git-ssh", secret: { secretName: registry.controller.source.githubSsh.secretName, defaultMode: 0o400 } }, ], containers: [{ name: "gate", image: registry.controller.image, imagePullPolicy: "IfNotPresent", command: ["/bin/sh", "-lc", gateScript], env: [ { name: "GATE", value: options.gate }, { name: "FOLLOWER_ID", value: follower.id }, { name: "REPOSITORY", value: follower.source.repository }, { name: "SOURCE_BRANCH", value: follower.source.branch }, { name: "SNAPSHOT_PREFIX", value: follower.source.snapshotPrefix }, { name: "SOURCE_COMMIT", value: options.sourceCommit ?? "" }, { name: "REPO_PATH", value: follower.nativeStatus.source.repoPath }, { name: "GITOPS_BRANCH", value: gitopsBranch }, { name: "TEKTON_NAMESPACE", value: follower.nativeStatus.tekton?.namespace ?? "" }, { name: "PIPELINE_RUN_PREFIX", value: follower.nativeStatus.tekton?.pipelineRunPrefix ?? "" }, { name: "ARGO_NAMESPACE", value: follower.nativeStatus.argo?.namespace ?? "" }, { name: "ARGO_APPLICATION", value: follower.nativeStatus.argo?.application ?? "" }, { name: "RUNTIME_NAMESPACE", value: follower.nativeStatus.runtime?.namespace ?? "" }, { name: "STATE_NAMESPACE", value: registry.controller.namespace }, { name: "STATE_CONFIGMAP", value: registry.controller.stateConfigMapName }, { name: "WORKLOADS_B64", value: Buffer.from(JSON.stringify(workloads), "utf8").toString("base64") }, { name: "HEALTH_URL", value: healthUrl }, { name: "SLOW_TASK_SECONDS", value: String(gatePolicy.slowTaskSeconds) }, { name: "HEALTH_TIMEOUT_MS", value: String(gatePolicy.healthTimeoutMs) }, { name: "GATE_TIMEOUT_MS", value: String(timeoutSeconds * 1000) }, { name: "UNIDESK_CONTROLLER_GITHUB_SSH_PRIVATE_KEY", value: `/git-ssh/${registry.controller.source.githubSsh.privateKeySecretKey}` }, { name: "UNIDESK_CONTROLLER_GITHUB_PROXY_HOST", value: registry.controller.source.githubSsh.proxyHost }, { name: "UNIDESK_CONTROLLER_GITHUB_PROXY_PORT", value: String(registry.controller.source.githubSsh.proxyPort) }, ], volumeMounts: [ { name: "registry", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true }, { name: "git-mirror-cache", mountPath: "/cache" }, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, ], }], }, }, }, }; } function nativeCapabilityJobName(followerId: string, action: string, sha: string): string { const prefix = `${safeName(followerId)}-${safeName(action)}`; return `${prefix}-${sha.slice(0, 12)}`.replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 63); } function gateHealthUrl(follower: FollowerSpec): string { if (follower.adapter === "agentrun-yaml-lane") { return resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }).spec.runtime.internalBaseUrl; } if (follower.adapter === "hwlab-node-runtime") { const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node); return `${spec.publicApiUrl.replace(/\/+$/u, "")}/health`; } return ""; } function gatePolicyEnv(follower: FollowerSpec): { slowTaskSeconds: number; healthTimeoutMs: number } { if (follower.drillDown === null) { throw new Error(`follower ${follower.id} registry is missing drillDown policy`); } return { slowTaskSeconds: follower.drillDown.taskRunTimeoutSeconds, healthTimeoutMs: follower.drillDown.taskRunTimeoutSeconds * 1000, }; } function parseFirstJsonObject(text: string): Record | null { const start = text.indexOf("{"); if (start < 0) return null; let depth = 0; let inString = false; let escaped = false; for (let index = start; index < text.length; index += 1) { const char = text[index]; if (inString) { if (escaped) escaped = false; else if (char === "\\") escaped = true; else if (char === "\"") inString = false; } else if (char === "\"") inString = true; else if (char === "{") depth += 1; else if (char === "}" && --depth === 0) { try { const parsed = JSON.parse(text.slice(start, index + 1)) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; } catch { return null; } } } return null; } function parseLastJsonObject(text: string, predicate: (value: Record) => boolean = () => true): Record | null { let last: Record | null = null; let offset = 0; while (offset < text.length) { const start = text.indexOf("{", offset); if (start < 0) break; let depth = 0; let inString = false; let escaped = false; let foundEnd = false; for (let index = start; index < text.length; index += 1) { const char = text[index]; if (inString) { if (escaped) escaped = false; else if (char === "\\") escaped = true; else if (char === "\"") inString = false; } else if (char === "\"") inString = true; else if (char === "{") depth += 1; else if (char === "}" && --depth === 0) { try { const parsed = JSON.parse(text.slice(start, index + 1)) as unknown; if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) && predicate(parsed as Record)) last = parsed as Record; } catch { // Keep scanning because controller bootstrap may print non-contract JSON-like fragments. } offset = index + 1; foundEnd = true; break; } } if (!foundEnd) break; } return last; } function isControlPlaneRefreshResult(value: Record): boolean { return value.ok === true && value.status === "applied" && value.sourceAuthority === "k8s-git-mirror-snapshot"; } function safeName(value: string): string { return value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/-+/gu, "-").replace(/^-|-$/gu, "").slice(0, 32); } function tailText(text: string, maxChars: number): string { return text.length <= maxChars ? text : text.slice(text.length - maxChars); }