Codex
2277 lines
99 KiB
TypeScript
2277 lines
99 KiB
TypeScript
// SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower.
|
|
// Responsibility: YAML-first K8s branch-follower controller, status, and adapter orchestration.
|
|
import { createHash } from "node:crypto";
|
|
import { readFileSync } from "node:fs";
|
|
import { isAbsolute } from "node:path";
|
|
import { repoRoot, rootPath, type UniDeskConfig } from "./config";
|
|
import { runCommand, type CommandResult } from "./command";
|
|
import { startJob } from "./jobs";
|
|
import type { RenderedCliResult } from "./output";
|
|
import { transPath } from "./hwlab-node/runtime-common";
|
|
import { configRefGraph, resolveConfigRefString } from "./ops/config-refs";
|
|
import {
|
|
arrayField,
|
|
asRecord,
|
|
booleanField,
|
|
integerField,
|
|
readYamlRecord,
|
|
recordField,
|
|
redactText,
|
|
shQuote,
|
|
stringArrayField,
|
|
stringField,
|
|
} from "./platform-infra-ops-library";
|
|
|
|
const DEFAULT_CONFIG_PATH = "config/cicd-branch-followers.yaml";
|
|
const SPEC_REF = "PJ2026-01060703";
|
|
const SPEC_VERSION = "draft-2026-07-03-p0-branch-follower";
|
|
|
|
type OutputMode = "human" | "json" | "yaml";
|
|
type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "events" | "logs";
|
|
type BranchFollowerPhase =
|
|
| "Observed"
|
|
| "Noop"
|
|
| "PendingTrigger"
|
|
| "Triggering"
|
|
| "ClosingOut"
|
|
| "Succeeded"
|
|
| "Failed"
|
|
| "Superseded"
|
|
| "Blocked"
|
|
| "Skipped";
|
|
|
|
interface ParsedOptions {
|
|
action: BranchFollowerAction;
|
|
configPath: string;
|
|
followerId: string | null;
|
|
all: boolean;
|
|
confirm: boolean;
|
|
dryRun: boolean;
|
|
wait: boolean;
|
|
controller: boolean;
|
|
live: boolean;
|
|
noLive: boolean;
|
|
full: boolean;
|
|
raw: boolean;
|
|
recordState: boolean;
|
|
output: OutputMode;
|
|
limit: number;
|
|
tailBytes: number;
|
|
timeoutSeconds: number | null;
|
|
}
|
|
|
|
interface CommandSpec {
|
|
argv: string[];
|
|
timeoutSeconds: number;
|
|
}
|
|
|
|
interface FollowerSpec {
|
|
id: string;
|
|
enabled: boolean;
|
|
adapter: string;
|
|
description: string;
|
|
source: {
|
|
repository: string;
|
|
branch: string;
|
|
branchRef: string;
|
|
authorityRef: string;
|
|
snapshotPrefix: string;
|
|
snapshotRef: string;
|
|
};
|
|
target: {
|
|
node: string;
|
|
lane: string;
|
|
namespace: string;
|
|
sentinel: string | null;
|
|
configRefs: Record<string, string>;
|
|
};
|
|
budgets: {
|
|
endToEndSeconds: number;
|
|
statusSeconds: number;
|
|
triggerSeconds: number;
|
|
sourceSyncSeconds: number;
|
|
};
|
|
commands: {
|
|
plan: CommandSpec;
|
|
status: CommandSpec;
|
|
trigger: CommandSpec;
|
|
events: CommandSpec;
|
|
logs: CommandSpec;
|
|
};
|
|
nativeStatus: NativeStatusSpec;
|
|
closeoutChecks: string[];
|
|
}
|
|
|
|
interface NativeStatusSpec {
|
|
source: {
|
|
gitMirrorReadUrl: string;
|
|
gitMirrorNamespace: string;
|
|
gitMirrorDeployment: string;
|
|
repoPath: string;
|
|
};
|
|
tekton: {
|
|
namespace: string;
|
|
pipelineRunPrefix: string;
|
|
} | null;
|
|
argo: {
|
|
namespace: string;
|
|
application: string;
|
|
} | null;
|
|
runtime: {
|
|
namespace: string;
|
|
workloads: NativeWorkloadSpec[];
|
|
} | null;
|
|
}
|
|
|
|
interface NativeWorkloadSpec {
|
|
kind: "Deployment" | "StatefulSet";
|
|
name: string;
|
|
sourceCommit: {
|
|
labels: string[];
|
|
annotations: string[];
|
|
podLabels: string[];
|
|
podAnnotations: string[];
|
|
env: string[];
|
|
};
|
|
}
|
|
|
|
interface ControllerSpec {
|
|
namespace: string;
|
|
kubeRoute: string;
|
|
fieldManager: string;
|
|
serviceAccountName: string;
|
|
deploymentName: string;
|
|
configMapName: string;
|
|
stateConfigMapName: string;
|
|
leaseName: string;
|
|
image: string;
|
|
labels: Record<string, string>;
|
|
source: {
|
|
repository: string;
|
|
branch: string;
|
|
gitMirrorReadUrl: string;
|
|
gitMirrorCachePvcName: string;
|
|
githubSsh: {
|
|
secretName: string;
|
|
privateKeySecretKey: string;
|
|
proxyHost: string;
|
|
proxyPort: number;
|
|
};
|
|
sourceAuthority: {
|
|
mode: string;
|
|
resolver: string;
|
|
allowHostGit: boolean;
|
|
allowHostWorkspace: boolean;
|
|
allowGithubDirectInPipeline: boolean;
|
|
};
|
|
sourceSnapshot: {
|
|
stageRefPrefix: string;
|
|
missingObjectPolicy: string;
|
|
refreshPolicy: string;
|
|
};
|
|
};
|
|
loop: {
|
|
intervalSeconds: number;
|
|
reconcileTimeoutSeconds: number;
|
|
};
|
|
budgets: {
|
|
applyWaitSeconds: number;
|
|
statusSeconds: number;
|
|
runOnceSeconds: number;
|
|
};
|
|
}
|
|
|
|
interface BranchFollowerRegistry {
|
|
path: string;
|
|
rawText: string;
|
|
rawSha256: string;
|
|
metadata: {
|
|
id: string;
|
|
owner: string;
|
|
specRef: string;
|
|
version: string;
|
|
};
|
|
controller: ControllerSpec;
|
|
followers: FollowerSpec[];
|
|
}
|
|
|
|
interface AdapterSummary {
|
|
ok: boolean;
|
|
command: string;
|
|
exitCode: number | null;
|
|
timedOut: boolean;
|
|
observedSha: string | null;
|
|
targetSha: string | null;
|
|
lastTriggeredSha: string | null;
|
|
lastSucceededSha: string | null;
|
|
pipelineRun: string | null;
|
|
inFlightJob: string | null;
|
|
aligned: boolean | null;
|
|
phase: BranchFollowerPhase;
|
|
message: string;
|
|
payload: Record<string, unknown> | null;
|
|
stderrTail: string;
|
|
stdoutTail: string;
|
|
}
|
|
|
|
interface NativeObjectBundle {
|
|
ok: boolean;
|
|
source: Record<string, unknown> | null;
|
|
pipelineRun: Record<string, unknown> | null;
|
|
argoApplication: Record<string, unknown> | null;
|
|
workloads: Record<string, unknown>[];
|
|
errors: string[];
|
|
exitCode: number | null;
|
|
timedOut: boolean;
|
|
stdoutTail: string;
|
|
stderrTail: string;
|
|
}
|
|
|
|
interface FollowerState {
|
|
id: string;
|
|
adapter: string;
|
|
enabled: boolean;
|
|
phase: BranchFollowerPhase;
|
|
source: {
|
|
repository: string;
|
|
branch: string;
|
|
branchRef: string;
|
|
snapshotPrefix: string;
|
|
observedSha: string | null;
|
|
};
|
|
target: {
|
|
node: string;
|
|
lane: string;
|
|
namespace: string;
|
|
sentinel: string | null;
|
|
targetSha: string | null;
|
|
};
|
|
lastTriggeredSha: string | null;
|
|
lastSucceededSha: string | null;
|
|
pipelineRun: string | null;
|
|
inFlightJob: string | null;
|
|
budgetSource: Record<string, number>;
|
|
controller: {
|
|
mode: "local-cli" | "k8s-controller";
|
|
stateConfigMap: string;
|
|
leaseName: string;
|
|
};
|
|
decision: string;
|
|
dryRun: boolean;
|
|
updatedAt: string;
|
|
warnings: string[];
|
|
next: Record<string, string>;
|
|
command?: Record<string, unknown>;
|
|
}
|
|
|
|
interface K8sStateRead {
|
|
ok: boolean;
|
|
stateByFollower: Record<string, Record<string, unknown>>;
|
|
stateConfigMapPresent: boolean;
|
|
deployment: Record<string, unknown> | null;
|
|
lease: Record<string, unknown> | null;
|
|
pods: Record<string, unknown> | null;
|
|
errors: string[];
|
|
}
|
|
|
|
export function cicdHelp(): unknown {
|
|
return {
|
|
command: "cicd branch-follower plan|apply|status|run-once|events|logs",
|
|
output: "text by default; use --json, --raw, or -o json|yaml for machine output",
|
|
usage: [
|
|
"bun scripts/cli.ts cicd branch-follower plan",
|
|
"bun scripts/cli.ts cicd branch-follower apply --confirm --wait",
|
|
"bun scripts/cli.ts cicd branch-follower status",
|
|
"bun scripts/cli.ts cicd branch-follower status --live",
|
|
"bun scripts/cli.ts cicd branch-follower run-once --all --dry-run",
|
|
"bun scripts/cli.ts cicd branch-follower run-once --follower hwlab-jd01-v03 --confirm --wait",
|
|
"bun scripts/cli.ts cicd branch-follower events --follower agentrun-jd01-v02",
|
|
"bun scripts/cli.ts cicd branch-follower logs --follower web-probe-sentinel-master",
|
|
],
|
|
config: DEFAULT_CONFIG_PATH,
|
|
spec: `${SPEC_REF} ${SPEC_VERSION}`,
|
|
description: "Deploy and inspect the YAML-first Kubernetes branch follower that follows HWLAB v0.3, AgentRun v0.2, and the selected web-probe sentinel master lane without using host worktrees as source authority.",
|
|
};
|
|
}
|
|
|
|
export async function runCicdCommand(_config: UniDeskConfig | null, args: string[]): Promise<RenderedCliResult> {
|
|
const top = args[0];
|
|
if (top === undefined || isHelpToken(top)) return renderMachine("cicd", cicdHelp(), "json");
|
|
if (top !== "branch-follower") {
|
|
throw new Error("cicd usage: cicd branch-follower plan|apply|status|run-once|events|logs");
|
|
}
|
|
const options = parseOptions(args.slice(1));
|
|
const command = commandLabel(options);
|
|
if (options.action === "help") return renderMachine(command, cicdHelp(), "json");
|
|
const registry = readRegistry(options.configPath);
|
|
switch (options.action) {
|
|
case "plan":
|
|
return renderResult(command, buildPlan(registry, options), options);
|
|
case "apply":
|
|
return renderResult(command, await applyController(registry, options), options);
|
|
case "status":
|
|
return renderResult(command, await buildStatus(registry, options), options);
|
|
case "run-once":
|
|
return renderResult(command, await runOnce(registry, options), options);
|
|
case "events":
|
|
case "logs":
|
|
return renderResult(command, await runFollowerDrillDown(registry, options), options);
|
|
case "help":
|
|
return renderMachine(command, cicdHelp(), "json");
|
|
}
|
|
}
|
|
|
|
function parseOptions(args: string[]): ParsedOptions {
|
|
const actionToken = args[0];
|
|
if (actionToken === undefined || isHelpToken(actionToken)) {
|
|
return defaultOptions("help", args.slice(actionToken === undefined ? 0 : 1));
|
|
}
|
|
if (!["plan", "apply", "status", "run-once", "events", "logs"].includes(actionToken)) {
|
|
throw new Error(`cicd branch-follower unknown action: ${actionToken}`);
|
|
}
|
|
const action = actionToken as BranchFollowerAction;
|
|
const options = defaultOptions(action, []);
|
|
const rest = args.slice(1);
|
|
for (let index = 0; index < rest.length; index += 1) {
|
|
const arg = rest[index] ?? "";
|
|
if (isHelpToken(arg)) {
|
|
options.action = "help";
|
|
} else if (arg === "--config") {
|
|
options.configPath = valueOption(rest, ++index, arg);
|
|
} else if (arg === "--follower" || arg === "--target") {
|
|
options.followerId = simpleId(valueOption(rest, ++index, arg), arg);
|
|
} else if (arg === "--all") {
|
|
options.all = true;
|
|
} else if (arg === "--confirm") {
|
|
options.confirm = true;
|
|
} else if (arg === "--dry-run") {
|
|
options.dryRun = true;
|
|
} else if (arg === "--wait") {
|
|
options.wait = true;
|
|
} else if (arg === "--controller") {
|
|
options.controller = true;
|
|
} else if (arg === "--live") {
|
|
options.live = true;
|
|
} else if (arg === "--no-live") {
|
|
options.noLive = true;
|
|
} else if (arg === "--full") {
|
|
options.full = true;
|
|
} else if (arg === "--raw" || arg === "--json") {
|
|
options.raw = true;
|
|
options.full = true;
|
|
options.output = "json";
|
|
} else if (arg === "--record-state") {
|
|
options.recordState = true;
|
|
} else if (arg === "-o" || arg === "--output") {
|
|
const value = valueOption(rest, ++index, arg);
|
|
if (value !== "json" && value !== "yaml" && value !== "wide" && value !== "text") throw new Error(`${arg} must be json, yaml, wide, or text`);
|
|
options.output = value === "wide" || value === "text" ? "human" : value;
|
|
if (value === "json" || value === "yaml") {
|
|
options.raw = true;
|
|
options.full = true;
|
|
}
|
|
} else if (arg === "--limit") {
|
|
options.limit = positiveInt(valueOption(rest, ++index, arg), arg);
|
|
} else if (arg === "--tail-bytes" || arg === "--tail") {
|
|
options.tailBytes = positiveInt(valueOption(rest, ++index, arg), arg);
|
|
} else if (arg === "--timeout-seconds") {
|
|
options.timeoutSeconds = positiveInt(valueOption(rest, ++index, arg), arg);
|
|
} else {
|
|
throw new Error(`unsupported cicd branch-follower option: ${arg}`);
|
|
}
|
|
}
|
|
if (options.confirm && options.dryRun) throw new Error("cicd branch-follower accepts only one of --confirm or --dry-run");
|
|
if (options.action === "apply" && !options.confirm) options.dryRun = true;
|
|
if (options.action === "run-once" && !options.confirm) options.dryRun = true;
|
|
if (options.action === "run-once" && options.confirm && !options.all && options.followerId === null) {
|
|
throw new Error("run-once --confirm requires --all or --follower <id>");
|
|
}
|
|
return options;
|
|
}
|
|
|
|
function defaultOptions(action: BranchFollowerAction, _args: string[]): ParsedOptions {
|
|
return {
|
|
action,
|
|
configPath: DEFAULT_CONFIG_PATH,
|
|
followerId: null,
|
|
all: false,
|
|
confirm: false,
|
|
dryRun: false,
|
|
wait: false,
|
|
controller: false,
|
|
live: false,
|
|
noLive: false,
|
|
full: false,
|
|
raw: false,
|
|
recordState: false,
|
|
output: "human",
|
|
limit: 20,
|
|
tailBytes: 12000,
|
|
timeoutSeconds: null,
|
|
};
|
|
}
|
|
|
|
function valueOption(args: string[], index: number, option: string): string {
|
|
const value = args[index];
|
|
if (value === undefined || value.length === 0 || value.startsWith("--")) throw new Error(`${option} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
function simpleId(value: string, option: string): string {
|
|
if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${option} must be a simple id`);
|
|
return value;
|
|
}
|
|
|
|
function positiveInt(value: string, option: string): number {
|
|
const parsed = Number(value);
|
|
if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${option} must be a positive integer`);
|
|
return parsed;
|
|
}
|
|
|
|
function isHelpToken(value: string): boolean {
|
|
return value === "-h" || value === "--help" || value === "help";
|
|
}
|
|
|
|
function readRegistry(configPath: string): BranchFollowerRegistry {
|
|
const absolute = isAbsolute(configPath) ? configPath : rootPath(configPath);
|
|
const rawText = readFileSync(absolute, "utf8");
|
|
const root = readYamlRecord<Record<string, unknown>>(absolute, "CicdBranchFollowerRegistry");
|
|
const metadata = recordField(root, "metadata", configPath);
|
|
const controller = parseController(recordField(root, "controller", configPath));
|
|
const followers = arrayField(root, "followers", configPath).map(parseFollower);
|
|
const ids = new Set<string>();
|
|
for (const follower of followers) {
|
|
if (ids.has(follower.id)) throw new Error(`${configPath}.followers has duplicate id ${follower.id}`);
|
|
ids.add(follower.id);
|
|
}
|
|
return {
|
|
path: configPath,
|
|
rawText,
|
|
rawSha256: createHash("sha256").update(rawText).digest("hex"),
|
|
metadata: {
|
|
id: stringField(metadata, "id", `${configPath}.metadata`),
|
|
owner: stringField(metadata, "owner", `${configPath}.metadata`),
|
|
specRef: stringField(metadata, "specRef", `${configPath}.metadata`),
|
|
version: stringField(metadata, "version", `${configPath}.metadata`),
|
|
},
|
|
controller,
|
|
followers,
|
|
};
|
|
}
|
|
|
|
function parseController(root: Record<string, unknown>): ControllerSpec {
|
|
const source = recordField(root, "source", "controller");
|
|
const authority = recordField(source, "sourceAuthority", "controller.source");
|
|
const snapshot = recordField(source, "sourceSnapshot", "controller.source");
|
|
const loop = recordField(root, "loop", "controller");
|
|
const budgets = recordField(root, "budgets", "controller");
|
|
const result: ControllerSpec = {
|
|
namespace: stringField(root, "namespace", "controller"),
|
|
kubeRoute: stringField(root, "kubeRoute", "controller"),
|
|
fieldManager: stringField(root, "fieldManager", "controller"),
|
|
serviceAccountName: stringField(root, "serviceAccountName", "controller"),
|
|
deploymentName: stringField(root, "deploymentName", "controller"),
|
|
configMapName: stringField(root, "configMapName", "controller"),
|
|
stateConfigMapName: stringField(root, "stateConfigMapName", "controller"),
|
|
leaseName: stringField(root, "leaseName", "controller"),
|
|
image: stringField(root, "image", "controller"),
|
|
labels: stringMap(recordField(root, "labels", "controller"), "controller.labels"),
|
|
source: {
|
|
repository: stringField(source, "repository", "controller.source"),
|
|
branch: stringField(source, "branch", "controller.source"),
|
|
gitMirrorReadUrl: stringField(source, "gitMirrorReadUrl", "controller.source"),
|
|
gitMirrorCachePvcName: stringField(source, "gitMirrorCachePvcName", "controller.source"),
|
|
githubSsh: parseControllerGithubSsh(recordField(source, "githubSsh", "controller.source")),
|
|
sourceAuthority: {
|
|
mode: stringField(authority, "mode", "controller.source.sourceAuthority"),
|
|
resolver: stringField(authority, "resolver", "controller.source.sourceAuthority"),
|
|
allowHostGit: booleanField(authority, "allowHostGit", "controller.source.sourceAuthority"),
|
|
allowHostWorkspace: booleanField(authority, "allowHostWorkspace", "controller.source.sourceAuthority"),
|
|
allowGithubDirectInPipeline: booleanField(authority, "allowGithubDirectInPipeline", "controller.source.sourceAuthority"),
|
|
},
|
|
sourceSnapshot: {
|
|
stageRefPrefix: stringField(snapshot, "stageRefPrefix", "controller.source.sourceSnapshot"),
|
|
missingObjectPolicy: stringField(snapshot, "missingObjectPolicy", "controller.source.sourceSnapshot"),
|
|
refreshPolicy: stringField(snapshot, "refreshPolicy", "controller.source.sourceSnapshot"),
|
|
},
|
|
},
|
|
loop: {
|
|
intervalSeconds: integerField(loop, "intervalSeconds", "controller.loop"),
|
|
reconcileTimeoutSeconds: integerField(loop, "reconcileTimeoutSeconds", "controller.loop"),
|
|
},
|
|
budgets: {
|
|
applyWaitSeconds: integerField(budgets, "applyWaitSeconds", "controller.budgets"),
|
|
statusSeconds: integerField(budgets, "statusSeconds", "controller.budgets"),
|
|
runOnceSeconds: integerField(budgets, "runOnceSeconds", "controller.budgets"),
|
|
},
|
|
};
|
|
if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) {
|
|
throw new Error("controller.source.sourceAuthority must disable host git, host workspace, and direct GitHub pipeline fallback");
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function parseControllerGithubSsh(root: Record<string, unknown>): ControllerSpec["source"]["githubSsh"] {
|
|
return {
|
|
secretName: stringField(root, "secretName", "controller.source.githubSsh"),
|
|
privateKeySecretKey: stringField(root, "privateKeySecretKey", "controller.source.githubSsh"),
|
|
proxyHost: stringField(root, "proxyHost", "controller.source.githubSsh"),
|
|
proxyPort: integerField(root, "proxyPort", "controller.source.githubSsh"),
|
|
};
|
|
}
|
|
|
|
function parseFollower(root: Record<string, unknown>, index: number): FollowerSpec {
|
|
const label = `followers[${index}]`;
|
|
const source = recordField(root, "source", label);
|
|
const target = recordField(root, "target", label);
|
|
const budgets = recordField(root, "budgets", label);
|
|
const commands = recordField(root, "commands", label);
|
|
const nativeStatus = recordField(root, "nativeStatus", label);
|
|
const closeout = recordField(root, "closeout", label);
|
|
const configRefs = stringMap(recordField(target, "configRefs", `${label}.target`), `${label}.target.configRefs`);
|
|
return {
|
|
id: simpleId(stringField(root, "id", label), `${label}.id`),
|
|
enabled: booleanField(root, "enabled", label),
|
|
adapter: stringField(root, "adapter", label),
|
|
description: stringField(root, "description", label),
|
|
source: {
|
|
repository: stringField(source, "repository", `${label}.source`),
|
|
branch: stringField(source, "branch", `${label}.source`),
|
|
branchRef: stringField(source, "branchRef", `${label}.source`),
|
|
authorityRef: stringField(source, "authorityRef", `${label}.source`),
|
|
snapshotPrefix: stringField(source, "snapshotPrefix", `${label}.source`),
|
|
snapshotRef: stringField(source, "snapshotRef", `${label}.source`),
|
|
},
|
|
target: {
|
|
node: stringField(target, "node", `${label}.target`),
|
|
lane: stringField(target, "lane", `${label}.target`),
|
|
namespace: stringField(target, "namespace", `${label}.target`),
|
|
sentinel: typeof target.sentinel === "string" && target.sentinel.length > 0 ? target.sentinel : null,
|
|
configRefs,
|
|
},
|
|
budgets: {
|
|
endToEndSeconds: integerField(budgets, "endToEndSeconds", `${label}.budgets`),
|
|
statusSeconds: integerField(budgets, "statusSeconds", `${label}.budgets`),
|
|
triggerSeconds: integerField(budgets, "triggerSeconds", `${label}.budgets`),
|
|
sourceSyncSeconds: integerField(budgets, "sourceSyncSeconds", `${label}.budgets`),
|
|
},
|
|
commands: {
|
|
plan: parseCommand(recordField(commands, "plan", `${label}.commands`), `${label}.commands.plan`),
|
|
status: parseCommand(recordField(commands, "status", `${label}.commands`), `${label}.commands.status`),
|
|
trigger: parseCommand(recordField(commands, "trigger", `${label}.commands`), `${label}.commands.trigger`),
|
|
events: parseCommand(recordField(commands, "events", `${label}.commands`), `${label}.commands.events`),
|
|
logs: parseCommand(recordField(commands, "logs", `${label}.commands`), `${label}.commands.logs`),
|
|
},
|
|
nativeStatus: parseNativeStatus(nativeStatus, `${label}.nativeStatus`),
|
|
closeoutChecks: stringArrayField(closeout, "checks", `${label}.closeout`),
|
|
};
|
|
}
|
|
|
|
function parseNativeStatus(root: Record<string, unknown>, label: string): NativeStatusSpec {
|
|
const source = recordField(root, "source", label);
|
|
const tekton = asOptionalRecord(root.tekton);
|
|
const argo = asOptionalRecord(root.argo);
|
|
const runtime = asOptionalRecord(root.runtime);
|
|
return {
|
|
source: {
|
|
gitMirrorReadUrl: stringField(source, "gitMirrorReadUrl", `${label}.source`),
|
|
gitMirrorNamespace: stringField(source, "gitMirrorNamespace", `${label}.source`),
|
|
gitMirrorDeployment: stringField(source, "gitMirrorDeployment", `${label}.source`),
|
|
repoPath: stringField(source, "repoPath", `${label}.source`),
|
|
},
|
|
tekton: tekton === null
|
|
? null
|
|
: {
|
|
namespace: stringField(tekton, "namespace", `${label}.tekton`),
|
|
pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${label}.tekton`),
|
|
},
|
|
argo: argo === null
|
|
? null
|
|
: {
|
|
namespace: stringField(argo, "namespace", `${label}.argo`),
|
|
application: stringField(argo, "application", `${label}.argo`),
|
|
},
|
|
runtime: runtime === null
|
|
? null
|
|
: {
|
|
namespace: stringField(runtime, "namespace", `${label}.runtime`),
|
|
workloads: arrayField(runtime, "workloads", `${label}.runtime`).map((item, index) => parseNativeWorkload(item, `${label}.runtime.workloads[${index}]`)),
|
|
},
|
|
};
|
|
}
|
|
|
|
function parseNativeWorkload(value: unknown, label: string): NativeWorkloadSpec {
|
|
const root = asRecord(value, label);
|
|
const kind = stringField(root, "kind", label);
|
|
if (kind !== "Deployment" && kind !== "StatefulSet") throw new Error(`${label}.kind must be Deployment or StatefulSet`);
|
|
const sourceCommit = asOptionalRecord(root.sourceCommit) ?? {};
|
|
return {
|
|
kind,
|
|
name: stringField(root, "name", label),
|
|
sourceCommit: {
|
|
labels: optionalStringArrayField(sourceCommit, "labels", `${label}.sourceCommit`),
|
|
annotations: optionalStringArrayField(sourceCommit, "annotations", `${label}.sourceCommit`),
|
|
podLabels: optionalStringArrayField(sourceCommit, "podLabels", `${label}.sourceCommit`),
|
|
podAnnotations: optionalStringArrayField(sourceCommit, "podAnnotations", `${label}.sourceCommit`),
|
|
env: optionalStringArrayField(sourceCommit, "env", `${label}.sourceCommit`),
|
|
},
|
|
};
|
|
}
|
|
|
|
function parseCommand(root: Record<string, unknown>, label: string): CommandSpec {
|
|
return {
|
|
argv: stringArrayField(root, "argv", label),
|
|
timeoutSeconds: integerField(root, "timeoutSeconds", label),
|
|
};
|
|
}
|
|
|
|
function optionalStringArrayField(root: Record<string, unknown>, key: string, label: string): string[] {
|
|
return root[key] === undefined ? [] : stringArrayField(root, key, label);
|
|
}
|
|
|
|
function stringMap(root: Record<string, unknown>, label: string): Record<string, string> {
|
|
const result: Record<string, string> = {};
|
|
for (const [key, value] of Object.entries(root)) {
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${label}.${key} must be a non-empty string`);
|
|
result[key] = value;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function buildPlan(registry: BranchFollowerRegistry, options: ParsedOptions): Record<string, unknown> {
|
|
const selected = selectFollowers(registry, options, { includeDisabled: true });
|
|
const followers = selected.map((follower) => {
|
|
const branchValue = safeResolveString(follower.source.branchRef);
|
|
const graph = configRefGraph([
|
|
{ id: "source.branch", ref: follower.source.branchRef },
|
|
{ id: "source.authority", ref: follower.source.authorityRef },
|
|
{ id: "source.snapshot", ref: follower.source.snapshotRef },
|
|
...Object.entries(follower.target.configRefs).map(([id, ref]) => ({ id: `target.${id}`, ref })),
|
|
]);
|
|
const warnings: string[] = [];
|
|
if (branchValue !== null && branchValue !== follower.source.branch) warnings.push(`source.branch ${follower.source.branch} differs from ${follower.source.branchRef} value ${branchValue}`);
|
|
return {
|
|
id: follower.id,
|
|
enabled: follower.enabled,
|
|
adapter: follower.adapter,
|
|
description: follower.description,
|
|
source: {
|
|
repository: follower.source.repository,
|
|
branch: follower.source.branch,
|
|
branchRef: follower.source.branchRef,
|
|
resolvedBranch: branchValue,
|
|
snapshotPrefix: follower.source.snapshotPrefix,
|
|
},
|
|
target: follower.target,
|
|
budgets: follower.budgets,
|
|
commands: redactCommands(follower),
|
|
nativeStatus: nativeStatusPlan(follower.nativeStatus),
|
|
closeoutChecks: follower.closeoutChecks,
|
|
configRefGraph: graph,
|
|
warnings,
|
|
};
|
|
});
|
|
return {
|
|
ok: true,
|
|
action: "plan",
|
|
spec: `${SPEC_REF} ${SPEC_VERSION}`,
|
|
registry: registrySummary(registry),
|
|
hostWorktreeAuthority: false,
|
|
sourceAuthority: {
|
|
mode: registry.controller.source.sourceAuthority.mode,
|
|
resolver: registry.controller.source.sourceAuthority.resolver,
|
|
allowHostGit: registry.controller.source.sourceAuthority.allowHostGit,
|
|
allowHostWorkspace: registry.controller.source.sourceAuthority.allowHostWorkspace,
|
|
allowGithubDirectInPipeline: registry.controller.source.sourceAuthority.allowGithubDirectInPipeline,
|
|
},
|
|
controller: {
|
|
namespace: registry.controller.namespace,
|
|
kubeRoute: registry.controller.kubeRoute,
|
|
deploymentName: registry.controller.deploymentName,
|
|
stateConfigMapName: registry.controller.stateConfigMapName,
|
|
leaseName: registry.controller.leaseName,
|
|
image: registry.controller.image,
|
|
loop: registry.controller.loop,
|
|
budgets: registry.controller.budgets,
|
|
},
|
|
followers,
|
|
next: {
|
|
apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait",
|
|
status: "bun scripts/cli.ts cicd branch-follower status",
|
|
dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run",
|
|
},
|
|
};
|
|
}
|
|
|
|
async function applyController(registry: BranchFollowerRegistry, options: ParsedOptions): Promise<Record<string, unknown>> {
|
|
const manifests = renderControllerManifests(registry);
|
|
const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
|
|
const manifestBase64 = Buffer.from(manifestYaml, "utf8").toString("base64");
|
|
const waitSeconds = options.timeoutSeconds ?? registry.controller.budgets.applyWaitSeconds;
|
|
const script = [
|
|
"set -eu",
|
|
"tmp=$(mktemp)",
|
|
"base64 -d >\"$tmp\" <<'UNIDESK_CICD_BRANCH_FOLLOWER_MANIFEST_B64'",
|
|
manifestBase64,
|
|
"UNIDESK_CICD_BRANCH_FOLLOWER_MANIFEST_B64",
|
|
options.dryRun
|
|
? `kubectl apply --dry-run=server --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp"`
|
|
: `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp"`,
|
|
!options.dryRun && options.wait
|
|
? `kubectl -n ${shQuote(registry.controller.namespace)} rollout status deploy/${shQuote(registry.controller.deploymentName)} --timeout=${waitSeconds}s`
|
|
: "true",
|
|
`kubectl -n ${shQuote(registry.controller.namespace)} get deploy/${shQuote(registry.controller.deploymentName)} cm/${shQuote(registry.controller.configMapName)} cm/${shQuote(registry.controller.stateConfigMapName)} lease/${shQuote(registry.controller.leaseName)} -o wide 2>/dev/null || true`,
|
|
].join("\n");
|
|
const result = runKubeScript(registry, options, script, "", (waitSeconds + 15) * 1000);
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
action: "apply",
|
|
dryRun: options.dryRun,
|
|
wait: options.wait,
|
|
registry: registrySummary(registry),
|
|
objects: manifests.map((item) => objectRef(item)),
|
|
manifestSha256: createHash("sha256").update(manifestYaml).digest("hex"),
|
|
controller: {
|
|
namespace: registry.controller.namespace,
|
|
route: registry.controller.kubeRoute,
|
|
deploymentName: registry.controller.deploymentName,
|
|
stateConfigMapName: registry.controller.stateConfigMapName,
|
|
leaseName: registry.controller.leaseName,
|
|
hostWorktreeMounted: false,
|
|
sourceMode: "k8s-git-mirror-to-emptyDir",
|
|
},
|
|
command: commandCompact(result, options),
|
|
next: {
|
|
status: "bun scripts/cli.ts cicd branch-follower status",
|
|
logs: `bun scripts/cli.ts cicd branch-follower logs --follower ${registry.followers[0]?.id ?? "<id>"}`,
|
|
dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run",
|
|
},
|
|
};
|
|
}
|
|
|
|
async function buildStatus(registry: BranchFollowerRegistry, options: ParsedOptions): Promise<Record<string, unknown>> {
|
|
let k8s = readK8sState(registry, options);
|
|
const wantsLive = options.live || (!options.noLive && Object.keys(k8s.stateByFollower).length === 0);
|
|
const refresh = wantsLive && !options.controller ? runControllerReconcileJob(registry, options, { dryRun: true, wait: true, recordState: true }) : null;
|
|
if (refresh !== null) k8s = readK8sState(registry, options);
|
|
const shouldLive = wantsLive && options.controller;
|
|
const selected = selectFollowers(registry, options, { includeDisabled: true });
|
|
const followers = [];
|
|
for (const follower of selected) {
|
|
const stored = k8s.stateByFollower[follower.id] ?? {};
|
|
const live = shouldLive && follower.enabled ? await readAdapterStatus(registry, follower, options) : null;
|
|
followers.push(mergeFollowerStatus(registry, follower, stored, live, shouldLive));
|
|
}
|
|
return {
|
|
ok: k8s.ok && followers.every((item) => item.ok !== false),
|
|
action: "status",
|
|
live: shouldLive,
|
|
registry: registrySummary(registry),
|
|
controller: controllerStatusSummary(registry, k8s),
|
|
followers,
|
|
refresh,
|
|
errors: k8s.errors,
|
|
next: {
|
|
apply: "bun scripts/cli.ts cicd branch-follower apply --confirm --wait",
|
|
liveStatus: "bun scripts/cli.ts cicd branch-follower status --live",
|
|
dryRun: "bun scripts/cli.ts cicd branch-follower run-once --all --dry-run",
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runOnce(registry: BranchFollowerRegistry, options: ParsedOptions): Promise<Record<string, unknown>> {
|
|
if (!options.controller) {
|
|
const refresh = runControllerReconcileJob(registry, options, { dryRun: options.dryRun, wait: true, recordState: true });
|
|
const k8s = readK8sState(registry, options);
|
|
const selected = selectFollowers(registry, options, { includeDisabled: false });
|
|
return {
|
|
ok: refresh.ok,
|
|
action: "run-once",
|
|
dryRun: options.dryRun,
|
|
confirm: options.confirm,
|
|
wait: true,
|
|
controller: false,
|
|
execution: "k8s-native-reconcile-job",
|
|
registry: registrySummary(registry),
|
|
job: refresh,
|
|
followers: selected.map((follower) => mergeFollowerStatus(registry, follower, k8s.stateByFollower[follower.id] ?? {}, null, false)),
|
|
warnings: refresh.ok ? [] : [`reconcile job failed: ${refresh.message}`],
|
|
next: {
|
|
status: "bun scripts/cli.ts cicd branch-follower status",
|
|
liveStatus: "bun scripts/cli.ts cicd branch-follower status --live",
|
|
},
|
|
};
|
|
}
|
|
const selected = selectFollowers(registry, options, { includeDisabled: false });
|
|
const previous = readK8sState(registry, options);
|
|
const results: FollowerState[] = [];
|
|
const stateWriteWarnings: string[] = [];
|
|
for (const follower of selected) {
|
|
const oldState = previous.stateByFollower[follower.id] ?? {};
|
|
const live = await readAdapterStatus(registry, follower, options);
|
|
const state = await decideAndMaybeTrigger(registry, follower, oldState, live, options);
|
|
if (!options.dryRun || options.recordState) {
|
|
const write = writeFollowerState(registry, state, options);
|
|
if (write.exitCode !== 0) {
|
|
const warning = `state write failed for ${follower.id}: ${tailText(write.stderr || write.stdout, 300)}`;
|
|
state.warnings.push(warning);
|
|
stateWriteWarnings.push(warning);
|
|
}
|
|
}
|
|
results.push(state);
|
|
}
|
|
return {
|
|
ok: results.every((item) => item.phase !== "Failed" && item.phase !== "Blocked"),
|
|
action: "run-once",
|
|
dryRun: options.dryRun,
|
|
confirm: options.confirm,
|
|
wait: options.wait,
|
|
controller: options.controller,
|
|
registry: registrySummary(registry),
|
|
followers: results,
|
|
warnings: stateWriteWarnings,
|
|
next: {
|
|
status: "bun scripts/cli.ts cicd branch-follower status",
|
|
liveStatus: "bun scripts/cli.ts cicd branch-follower status --live",
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runFollowerDrillDown(registry: BranchFollowerRegistry, options: ParsedOptions): Promise<Record<string, unknown>> {
|
|
if (options.followerId === null) {
|
|
return {
|
|
ok: true,
|
|
action: options.action,
|
|
message: "select one follower to inspect native Kubernetes status objects",
|
|
followers: registry.followers.map((follower) => ({
|
|
id: follower.id,
|
|
adapter: follower.adapter,
|
|
statusAuthority: "k8s-native",
|
|
nativeStatus: nativeStatusPlan(follower.nativeStatus),
|
|
})),
|
|
};
|
|
}
|
|
const follower = registry.followers.find((item) => item.id === options.followerId);
|
|
if (follower === undefined) throw new Error(`unknown follower ${options.followerId}`);
|
|
const live = await readAdapterStatus(registry, follower, options);
|
|
return {
|
|
ok: live.ok,
|
|
action: options.action,
|
|
follower: follower.id,
|
|
adapter: follower.adapter,
|
|
statusAuthority: "k8s-native",
|
|
parsedDownstreamCliOutput: false,
|
|
summary: {
|
|
phase: live.phase,
|
|
observedSha: live.observedSha,
|
|
targetSha: live.targetSha,
|
|
pipelineRun: live.pipelineRun,
|
|
aligned: live.aligned,
|
|
message: live.message,
|
|
},
|
|
native: live.payload,
|
|
next: {
|
|
status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`,
|
|
runOnceDryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function decideAndMaybeTrigger(
|
|
registry: BranchFollowerRegistry,
|
|
follower: FollowerSpec,
|
|
previous: Record<string, unknown>,
|
|
live: AdapterSummary,
|
|
options: ParsedOptions,
|
|
): Promise<FollowerState> {
|
|
const warnings: string[] = [];
|
|
if (!live.ok) warnings.push(`status command failed: exitCode=${live.exitCode}${live.timedOut ? " timedOut=true" : ""}`);
|
|
const observedSha = live.observedSha;
|
|
let targetSha = live.targetSha;
|
|
const previousLastTriggered = stringOrNull(previous.lastTriggeredSha);
|
|
const previousInFlight = stringOrNull(previous.inFlightJob);
|
|
const previousLastSucceeded = stringOrNull(previous.lastSucceededSha);
|
|
const previousObserved = stringOrNull(recordAt(previous, ["source"])?.observedSha);
|
|
const superseded = previousInFlight !== null && previousObserved !== null && observedSha !== null && previousObserved !== observedSha;
|
|
let phase: BranchFollowerPhase;
|
|
let decision: string;
|
|
let triggerCommand: Record<string, unknown> | undefined;
|
|
let inFlightJob: string | null = live.inFlightJob;
|
|
let lastTriggeredSha = live.lastTriggeredSha ?? previousLastTriggered;
|
|
let lastSucceededSha = live.lastSucceededSha ?? previousLastSucceeded;
|
|
if (targetSha === null && observedSha !== null && previousLastSucceeded === observedSha) targetSha = observedSha;
|
|
|
|
if (!follower.enabled) {
|
|
phase = "Skipped";
|
|
decision = "follower disabled";
|
|
} else if (observedSha === null) {
|
|
phase = live.ok ? "Observed" : "Blocked";
|
|
decision = "status did not expose an observed source sha; adapter trigger-current remains the dedupe authority";
|
|
} else if (superseded) {
|
|
phase = "Superseded";
|
|
decision = `previous in-flight sha ${shortSha(previousObserved)} was superseded by ${shortSha(observedSha)}`;
|
|
} else if (targetSha !== null && targetSha === observedSha) {
|
|
phase = "Noop";
|
|
decision = "target already matches observed source sha";
|
|
lastSucceededSha = observedSha;
|
|
} else if (previousLastTriggered !== null && previousLastTriggered === observedSha) {
|
|
phase = "ClosingOut";
|
|
decision = "same sha was already triggered; use status/events/logs for closeout";
|
|
} else {
|
|
phase = "PendingTrigger";
|
|
decision = targetSha === null
|
|
? "target sha is unknown; trigger-current adapter will dedupe by source snapshot"
|
|
: `observed ${shortSha(observedSha)} differs from target ${shortSha(targetSha)}`;
|
|
}
|
|
|
|
if (options.confirm && (phase === "PendingTrigger" || phase === "Superseded" || (phase === "Observed" && observedSha !== null))) {
|
|
const trigger = await executeTrigger(follower, observedSha, options);
|
|
triggerCommand = trigger.command;
|
|
phase = trigger.ok ? (options.wait || options.controller ? "ClosingOut" : "Triggering") : "Failed";
|
|
decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}`;
|
|
inFlightJob = trigger.jobId ?? live.inFlightJob;
|
|
lastTriggeredSha = observedSha;
|
|
if (trigger.ok && options.wait) {
|
|
targetSha = observedSha;
|
|
lastSucceededSha = observedSha;
|
|
}
|
|
if (!trigger.ok) warnings.push(trigger.message);
|
|
}
|
|
|
|
if (options.dryRun && phase === "PendingTrigger") decision = `${decision}; dry-run did not trigger`;
|
|
|
|
return {
|
|
id: follower.id,
|
|
adapter: follower.adapter,
|
|
enabled: follower.enabled,
|
|
phase,
|
|
source: {
|
|
repository: follower.source.repository,
|
|
branch: follower.source.branch,
|
|
branchRef: follower.source.branchRef,
|
|
snapshotPrefix: follower.source.snapshotPrefix,
|
|
observedSha,
|
|
},
|
|
target: {
|
|
node: follower.target.node,
|
|
lane: follower.target.lane,
|
|
namespace: follower.target.namespace,
|
|
sentinel: follower.target.sentinel,
|
|
targetSha,
|
|
},
|
|
lastTriggeredSha,
|
|
lastSucceededSha,
|
|
pipelineRun: live.pipelineRun,
|
|
inFlightJob,
|
|
budgetSource: follower.budgets,
|
|
controller: {
|
|
mode: options.controller ? "k8s-controller" : "local-cli",
|
|
stateConfigMap: registry.controller.stateConfigMapName,
|
|
leaseName: registry.controller.leaseName,
|
|
},
|
|
decision,
|
|
dryRun: options.dryRun,
|
|
updatedAt: new Date().toISOString(),
|
|
warnings,
|
|
next: followerNextCommands(follower),
|
|
command: triggerCommand ?? {
|
|
status: live.command,
|
|
exitCode: live.exitCode,
|
|
timedOut: live.timedOut,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function executeTrigger(follower: FollowerSpec, observedSha: string | null, options: ParsedOptions): Promise<{ ok: boolean; message: string; jobId: string | null; command: Record<string, unknown> }> {
|
|
const spec = follower.commands.trigger;
|
|
const timeoutSeconds = options.timeoutSeconds ?? spec.timeoutSeconds;
|
|
if (!options.wait && !options.controller) {
|
|
const job = startJob(`cicd_branch_follower_${safeJobSegment(follower.id)}`, spec.argv, `Trigger ${follower.id} for observed sha ${observedSha ?? "unknown"}`);
|
|
return {
|
|
ok: true,
|
|
message: `started async job ${job.id}`,
|
|
jobId: job.id,
|
|
command: {
|
|
mode: "async-job",
|
|
argv: spec.argv,
|
|
jobId: job.id,
|
|
status: `bun scripts/cli.ts job status ${job.id}`,
|
|
},
|
|
};
|
|
}
|
|
const result = runCommand(spec.argv, repoRoot, { timeoutMs: timeoutSeconds * 1000 });
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
message: result.exitCode === 0 ? "trigger command completed" : tailText(result.stderr || result.stdout, 500),
|
|
jobId: null,
|
|
command: commandCompact(result, options),
|
|
};
|
|
}
|
|
|
|
async function readAdapterStatus(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Promise<AdapterSummary> {
|
|
const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.statusSeconds;
|
|
const bundle = readNativeObjectBundle(registry, follower, options, timeoutSeconds);
|
|
const observedSha = stringOrNull(bundle.source?.commit);
|
|
const runtimeTargetSha = runtimeTargetShaFromWorkloads(follower.nativeStatus.runtime, bundle.workloads);
|
|
const pipelineRunName = stringOrNull(asOptionalRecord(bundle.pipelineRun?.metadata)?.name) ?? expectedPipelineRunName(follower, observedSha);
|
|
const pipelineSucceeded = pipelineRunSucceeded(bundle.pipelineRun);
|
|
const argoReady = follower.nativeStatus.argo === null ? null : argoApplicationReady(bundle.argoApplication);
|
|
const runtimeReady = follower.nativeStatus.runtime === null ? null : runtimeWorkloadsReady(follower.nativeStatus.runtime, bundle.workloads);
|
|
const hasTektonGate = follower.nativeStatus.tekton !== null;
|
|
const hasRuntimeTarget = runtimeTargetSha !== null;
|
|
const hasTargetEvidence = hasRuntimeTarget || hasTektonGate;
|
|
const pipelineGateOk = !hasTektonGate
|
|
|| pipelineSucceeded === true
|
|
|| (hasRuntimeTarget && runtimeTargetSha === observedSha && argoReady !== false && runtimeReady !== false);
|
|
const aligned = observedSha !== null
|
|
&& hasTargetEvidence
|
|
&& (hasRuntimeTarget ? runtimeTargetSha === observedSha : true)
|
|
&& pipelineGateOk
|
|
&& argoReady !== false
|
|
&& runtimeReady !== false;
|
|
const targetSha = hasRuntimeTarget && runtimeTargetSha === observedSha && aligned ? runtimeTargetSha : runtimeTargetSha;
|
|
const ok = bundle.ok;
|
|
const phase = inferPhase(ok, aligned, observedSha, targetSha, bundle.timedOut);
|
|
return {
|
|
ok,
|
|
command: "native:k8s-git-mirror+tekton+argocd+runtime",
|
|
exitCode: bundle.exitCode,
|
|
timedOut: bundle.timedOut,
|
|
observedSha,
|
|
targetSha,
|
|
lastTriggeredSha: null,
|
|
lastSucceededSha: aligned && observedSha !== null ? observedSha : null,
|
|
pipelineRun: pipelineRunName,
|
|
inFlightJob: pipelineSucceeded === null && pipelineRunName !== null ? pipelineRunName : null,
|
|
aligned,
|
|
phase,
|
|
message: nativeStatusMessage(ok, phase, observedSha, targetSha, {
|
|
pipelineSucceeded,
|
|
argoReady,
|
|
runtimeReady,
|
|
errors: bundle.errors,
|
|
}),
|
|
payload: {
|
|
source: bundle.source,
|
|
tekton: nativePipelineRunSummary(bundle.pipelineRun),
|
|
argo: nativeArgoSummary(bundle.argoApplication),
|
|
runtime: nativeRuntimeSummary(follower.nativeStatus.runtime, bundle.workloads),
|
|
errors: bundle.errors,
|
|
statusAuthority: "k8s-native",
|
|
parsedDownstreamCliOutput: false,
|
|
},
|
|
stderrTail: bundle.stderrTail,
|
|
stdoutTail: bundle.stdoutTail,
|
|
};
|
|
}
|
|
|
|
function inferPhase(ok: boolean, aligned: boolean | null, observedSha: string | null, targetSha: string | null, timedOut: boolean): BranchFollowerPhase {
|
|
if (!ok || timedOut) return "Blocked";
|
|
if (aligned === true) return "Succeeded";
|
|
if (observedSha !== null && targetSha !== null && observedSha === targetSha) return "Noop";
|
|
if (observedSha !== null) return "PendingTrigger";
|
|
return "Observed";
|
|
}
|
|
|
|
function nativeStatusMessage(ok: boolean, phase: BranchFollowerPhase, observedSha: string | null, targetSha: string | null, gates: { pipelineSucceeded: boolean | null; argoReady: boolean | null; runtimeReady: boolean | null; errors: string[] }): string {
|
|
if (!ok) return gates.errors[0] ?? "native status read failed";
|
|
if (phase === "Noop" || phase === "Succeeded") return `target matches ${shortSha(observedSha)}`;
|
|
if (observedSha !== null && targetSha !== null) return `observed ${shortSha(observedSha)} target ${shortSha(targetSha)}`;
|
|
if (observedSha !== null) {
|
|
const gatesText = [
|
|
gates.pipelineSucceeded === false ? "pipelineRun not successful" : null,
|
|
gates.argoReady === false ? "argo not healthy/synced" : null,
|
|
gates.runtimeReady === false ? "runtime not ready" : null,
|
|
].filter((item): item is string => item !== null).join("; ");
|
|
return gatesText.length > 0 ? `observed ${shortSha(observedSha)}; ${gatesText}` : `observed ${shortSha(observedSha)} target unknown`;
|
|
}
|
|
return "k8s-native status did not expose observed source sha";
|
|
}
|
|
|
|
function readNativeObjectBundle(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, timeoutSeconds: number): NativeObjectBundle {
|
|
const native = follower.nativeStatus;
|
|
const source = native.source;
|
|
const tekton = native.tekton;
|
|
const argo = native.argo;
|
|
const runtime = native.runtime;
|
|
const workloadCommands = (runtime?.workloads ?? []).map((workload, index) => {
|
|
const resource = workload.kind === "Deployment" ? "deployment" : "statefulset";
|
|
return `emit_json ${shQuote(`workload${index}`)} kubectl -n ${shQuote(runtime?.namespace ?? follower.target.namespace)} get ${resource} ${shQuote(workload.name)} -o json`;
|
|
});
|
|
const script = [
|
|
"set +e",
|
|
"tmpdir=$(mktemp -d)",
|
|
"cleanup() { rm -rf \"$tmpdir\"; }",
|
|
"trap cleanup EXIT INT TERM",
|
|
"cat >\"$tmpdir/compact-native-object.mjs\" <<'NODE_COMPACT'",
|
|
"import { readFileSync } from 'node:fs';",
|
|
"const key = process.argv[2] || '';",
|
|
"const input = JSON.parse(readFileSync(0, 'utf8'));",
|
|
"const cleanMap = (value) => {",
|
|
" if (!value || typeof value !== 'object' || Array.isArray(value)) return {};",
|
|
" const out = {};",
|
|
" for (const [k, v] of Object.entries(value)) {",
|
|
" if (k === 'kubectl.kubernetes.io/last-applied-configuration') continue;",
|
|
" out[k] = v;",
|
|
" }",
|
|
" return out;",
|
|
"};",
|
|
"const metadata = (obj) => ({",
|
|
" name: obj?.metadata?.name || null,",
|
|
" namespace: obj?.metadata?.namespace || null,",
|
|
" labels: cleanMap(obj?.metadata?.labels),",
|
|
" annotations: cleanMap(obj?.metadata?.annotations),",
|
|
"});",
|
|
"const compactContainer = (container) => ({",
|
|
" name: container?.name || null,",
|
|
" image: container?.image || null,",
|
|
" env: Array.isArray(container?.env) ? container.env.filter((item) => item && typeof item.name === 'string' && typeof item.value === 'string').map((item) => ({ name: item.name, value: item.value })) : [],",
|
|
"});",
|
|
"let output = input;",
|
|
"if (key === 'pipelineRun') {",
|
|
" output = {",
|
|
" apiVersion: input.apiVersion,",
|
|
" kind: input.kind,",
|
|
" metadata: metadata(input),",
|
|
" spec: { params: Array.isArray(input?.spec?.params) ? input.spec.params : [] },",
|
|
" status: { conditions: Array.isArray(input?.status?.conditions) ? input.status.conditions : [], startTime: input?.status?.startTime || null, completionTime: input?.status?.completionTime || null },",
|
|
" };",
|
|
"} else if (key === 'argoApplication') {",
|
|
" output = {",
|
|
" apiVersion: input.apiVersion,",
|
|
" kind: input.kind,",
|
|
" metadata: metadata(input),",
|
|
" status: { sync: input?.status?.sync || null, health: input?.status?.health || null, operationState: input?.status?.operationState ? { phase: input.status.operationState.phase || null, message: input.status.operationState.message || null, finishedAt: input.status.operationState.finishedAt || null } : null },",
|
|
" };",
|
|
"} else if (/^workload\\d+$/.test(key)) {",
|
|
" const template = input?.spec?.template || {};",
|
|
" output = {",
|
|
" apiVersion: input.apiVersion,",
|
|
" kind: input.kind,",
|
|
" metadata: metadata(input),",
|
|
" spec: { replicas: input?.spec?.replicas ?? null, template: { metadata: { labels: cleanMap(template?.metadata?.labels), annotations: cleanMap(template?.metadata?.annotations) }, spec: { containers: Array.isArray(template?.spec?.containers) ? template.spec.containers.map(compactContainer) : [] } } },",
|
|
" status: { replicas: input?.status?.replicas ?? null, readyReplicas: input?.status?.readyReplicas ?? null, availableReplicas: input?.status?.availableReplicas ?? null, updatedReplicas: input?.status?.updatedReplicas ?? null, conditions: Array.isArray(input?.status?.conditions) ? input.status.conditions.map((item) => ({ type: item.type || null, status: item.status || null, reason: item.reason || null })) : [] },",
|
|
" };",
|
|
"}",
|
|
"console.log(JSON.stringify(output));",
|
|
"NODE_COMPACT",
|
|
"emit_json() {",
|
|
" key=\"$1\"",
|
|
" shift",
|
|
" raw=\"$tmpdir/$key.raw\"",
|
|
" out=\"$tmpdir/$key.out\"",
|
|
" err=\"$tmpdir/$key.err\"",
|
|
" if \"$@\" >\"$raw\" 2>\"$err\" && node \"$tmpdir/compact-native-object.mjs\" \"$key\" <\"$raw\" >\"$out\" 2>>\"$err\"; then",
|
|
" printf 'UNIDESK_NATIVE_JSON\\t%s\\t' \"$key\"",
|
|
" base64 \"$out\" | tr -d '\\n'",
|
|
" printf '\\n'",
|
|
" else",
|
|
" printf 'UNIDESK_NATIVE_ERROR\\t%s\\t' \"$key\"",
|
|
" base64 \"$err\" | tr -d '\\n'",
|
|
" printf '\\n'",
|
|
" fi",
|
|
"}",
|
|
`repo_path=${shQuote(source.repoPath)}`,
|
|
`branch=${shQuote(follower.source.branch)}`,
|
|
`repository=${shQuote(follower.source.repository)}`,
|
|
`snapshot_prefix=${shQuote(follower.source.snapshotPrefix)}`,
|
|
`read_url=${shQuote(source.gitMirrorReadUrl)}`,
|
|
`mirror_ns=${shQuote(source.gitMirrorNamespace)}`,
|
|
`mirror_deploy=${shQuote(source.gitMirrorDeployment)}`,
|
|
"source_commit=",
|
|
"source_err=\"$tmpdir/source.err\"",
|
|
"if [ -x /etc/unidesk-cicd-branch-follower/sync-source.sh ]; then",
|
|
" /etc/unidesk-cicd-branch-follower/sync-source.sh \"$repository\" \"$branch\" \"$snapshot_prefix\" \"$repo_path\" >/dev/null 2>\"$source_err\" || true",
|
|
"fi",
|
|
"if [ -d \"$repo_path/objects\" ]; then",
|
|
" source_commit=$(git --git-dir=\"$repo_path\" rev-parse --verify \"refs/heads/$branch^{commit}\" 2>>\"$source_err\" | head -n 1 | tr -d '\\r' || true)",
|
|
"else",
|
|
" source_commit=$(kubectl -n \"$mirror_ns\" exec \"deploy/$mirror_deploy\" -- env REPO_PATH=\"$repo_path\" BRANCH=\"$branch\" sh -lc 'git --git-dir=\"$REPO_PATH\" rev-parse --verify \"refs/heads/$BRANCH^{commit}\"' 2>>\"$source_err\" | head -n 1 | tr -d '\\r' || true)",
|
|
"fi",
|
|
"case \"$source_commit\" in",
|
|
" [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f])",
|
|
" printf 'UNIDESK_NATIVE_JSON\\tsource\\t'",
|
|
" printf '{\"commit\":\"%s\",\"branch\":\"%s\",\"mode\":\"k8s-git-mirror-cache\",\"repoPath\":\"%s\"}' \"$source_commit\" \"$branch\" \"$repo_path\" | base64 | tr -d '\\n'",
|
|
" printf '\\n'",
|
|
" ;;",
|
|
" *)",
|
|
" printf 'UNIDESK_NATIVE_ERROR\\tsource\\t'",
|
|
" base64 \"$source_err\" | tr -d '\\n'",
|
|
" printf '\\n'",
|
|
" ;;",
|
|
"esac",
|
|
tekton === null
|
|
? "true"
|
|
: [
|
|
"if [ -n \"$source_commit\" ]; then",
|
|
" sha12=$(printf '%s' \"$source_commit\" | cut -c1-12)",
|
|
` emit_json pipelineRun kubectl -n ${shQuote(tekton.namespace)} get pipelinerun ${shQuote(`${tekton.pipelineRunPrefix}-`)}"$sha12" -o json`,
|
|
"fi",
|
|
].join("\n"),
|
|
argo === null
|
|
? "true"
|
|
: `emit_json argoApplication kubectl -n ${shQuote(argo.namespace)} get application ${shQuote(argo.application)} -o json`,
|
|
...workloadCommands,
|
|
"exit 0",
|
|
].join("\n");
|
|
const result = runKubeScript(registry, options, script, "", Math.max(5, timeoutSeconds) * 1000);
|
|
const parsed = parseNativeBundleLines(result.stdout);
|
|
const sourceRecord = asOptionalRecord(parsed.objects.source);
|
|
return {
|
|
ok: result.exitCode === 0 && sourceRecord !== null && parsed.fatalErrors.length === 0,
|
|
source: sourceRecord,
|
|
pipelineRun: asOptionalRecord(parsed.objects.pipelineRun),
|
|
argoApplication: asOptionalRecord(parsed.objects.argoApplication),
|
|
workloads: Object.entries(parsed.objects)
|
|
.filter(([key]) => /^workload\d+$/u.test(key))
|
|
.sort(([left], [right]) => left.localeCompare(right))
|
|
.map(([, value]) => asOptionalRecord(value))
|
|
.filter((item): item is Record<string, unknown> => item !== null),
|
|
errors: [
|
|
...parsed.errors,
|
|
...(result.exitCode === 0 ? [] : [`native bundle command failed: exitCode=${result.exitCode}`]),
|
|
...parsed.fatalErrors,
|
|
],
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
stdoutTail: redactText(tailText(result.stdout, 1000)),
|
|
stderrTail: redactText(tailText(result.stderr, 1000)),
|
|
};
|
|
}
|
|
|
|
function parseNativeBundleLines(stdout: string): { objects: Record<string, unknown>; errors: string[]; fatalErrors: string[] } {
|
|
const objects: Record<string, unknown> = {};
|
|
const errors: string[] = [];
|
|
const fatalErrors: string[] = [];
|
|
for (const line of stdout.split(/\r?\n/u)) {
|
|
if (!line.startsWith("UNIDESK_NATIVE_")) continue;
|
|
const [kind, key, payload] = line.split("\t");
|
|
if (kind === undefined || key === undefined || payload === undefined) continue;
|
|
const decoded = Buffer.from(payload, "base64").toString("utf8").trim();
|
|
if (kind === "UNIDESK_NATIVE_JSON") {
|
|
const parsed = parseJsonObject(decoded);
|
|
if (parsed !== null) objects[key] = parsed;
|
|
else errors.push(`${key}: invalid native JSON payload`);
|
|
} else if (kind === "UNIDESK_NATIVE_ERROR") {
|
|
const message = `${key}: ${redactText(tailText(decoded, 500)) || "not found"}`;
|
|
errors.push(message);
|
|
if (key === "source") fatalErrors.push(message);
|
|
}
|
|
}
|
|
return { objects, errors, fatalErrors };
|
|
}
|
|
|
|
function expectedPipelineRunName(follower: FollowerSpec, observedSha: string | null): string | null {
|
|
if (observedSha === null || follower.nativeStatus.tekton === null) return null;
|
|
return `${follower.nativeStatus.tekton.pipelineRunPrefix}-${shortSha(observedSha)}`;
|
|
}
|
|
|
|
function pipelineRunSucceeded(pipelineRun: Record<string, unknown> | null): boolean | null {
|
|
if (pipelineRun === null) return null;
|
|
const conditions = Array.isArray(asOptionalRecord(pipelineRun.status)?.conditions) ? asOptionalRecord(pipelineRun.status)?.conditions : [];
|
|
for (const condition of conditions as unknown[]) {
|
|
const record = asOptionalRecord(condition);
|
|
if (record?.type !== "Succeeded") continue;
|
|
if (record.status === "True") return true;
|
|
if (record.status === "False") return false;
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function argoApplicationReady(application: Record<string, unknown> | null): boolean {
|
|
if (application === null) return false;
|
|
const status = asOptionalRecord(application.status);
|
|
const sync = asOptionalRecord(status?.sync);
|
|
const health = asOptionalRecord(status?.health);
|
|
return sync?.status === "Synced" && health?.status === "Healthy";
|
|
}
|
|
|
|
function runtimeWorkloadsReady(runtime: NativeStatusSpec["runtime"], workloads: Record<string, unknown>[]): boolean {
|
|
if (runtime === null) return true;
|
|
if (workloads.length < runtime.workloads.length) return false;
|
|
return runtime.workloads.every((spec, index) => workloadReady(spec, workloads[index] ?? null));
|
|
}
|
|
|
|
function workloadReady(spec: NativeWorkloadSpec, workload: Record<string, unknown> | null): boolean {
|
|
if (workload === null) return false;
|
|
const status = asOptionalRecord(workload.status);
|
|
if (spec.kind === "Deployment") {
|
|
const desired = numberOrNull(asOptionalRecord(workload.spec)?.replicas) ?? 1;
|
|
const available = numberOrNull(status?.availableReplicas) ?? 0;
|
|
const updated = numberOrNull(status?.updatedReplicas) ?? 0;
|
|
return available >= desired && updated >= desired;
|
|
}
|
|
const desired = numberOrNull(asOptionalRecord(workload.spec)?.replicas) ?? 1;
|
|
const ready = numberOrNull(status?.readyReplicas) ?? 0;
|
|
return ready >= desired;
|
|
}
|
|
|
|
function runtimeTargetShaFromWorkloads(runtime: NativeStatusSpec["runtime"], workloads: Record<string, unknown>[]): string | null {
|
|
if (runtime === null) return null;
|
|
const commits: string[] = [];
|
|
runtime.workloads.forEach((spec, index) => {
|
|
const commit = workloadSourceCommit(spec, workloads[index] ?? null);
|
|
if (commit !== null) commits.push(commit);
|
|
});
|
|
const unique = Array.from(new Set(commits));
|
|
return unique.length === 1 ? unique[0] ?? null : null;
|
|
}
|
|
|
|
function workloadSourceCommit(spec: NativeWorkloadSpec, workload: Record<string, unknown> | null): string | null {
|
|
if (workload === null) return null;
|
|
const metadata = asOptionalRecord(workload.metadata);
|
|
const labels = asOptionalRecord(metadata?.labels);
|
|
const annotations = asOptionalRecord(metadata?.annotations);
|
|
const template = asOptionalRecord(asOptionalRecord(workload.spec)?.template);
|
|
const podMetadata = asOptionalRecord(template?.metadata);
|
|
const podLabels = asOptionalRecord(podMetadata?.labels);
|
|
const podAnnotations = asOptionalRecord(podMetadata?.annotations);
|
|
for (const key of spec.sourceCommit.labels) {
|
|
const value = shaOrNull(labels?.[key]);
|
|
if (value !== null) return value;
|
|
}
|
|
for (const key of spec.sourceCommit.annotations) {
|
|
const value = shaOrNull(annotations?.[key]);
|
|
if (value !== null) return value;
|
|
}
|
|
for (const key of spec.sourceCommit.podLabels) {
|
|
const value = shaOrNull(podLabels?.[key]);
|
|
if (value !== null) return value;
|
|
}
|
|
for (const key of spec.sourceCommit.podAnnotations) {
|
|
const value = shaOrNull(podAnnotations?.[key]);
|
|
if (value !== null) return value;
|
|
}
|
|
const containers = Array.isArray(asOptionalRecord(template?.spec)?.containers) ? asOptionalRecord(template?.spec)?.containers : [];
|
|
for (const envName of spec.sourceCommit.env) {
|
|
for (const container of containers as unknown[]) {
|
|
const env = Array.isArray(asOptionalRecord(container)?.env) ? asOptionalRecord(container)?.env : [];
|
|
for (const entry of env as unknown[]) {
|
|
const record = asOptionalRecord(entry);
|
|
if (record?.name !== envName) continue;
|
|
const value = shaOrNull(record.value);
|
|
if (value !== null) return value;
|
|
}
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function shaOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && /^[0-9a-f]{40}$/iu.test(value) ? value : null;
|
|
}
|
|
|
|
function nativePipelineRunSummary(pipelineRun: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (pipelineRun === null) return null;
|
|
const metadata = asOptionalRecord(pipelineRun.metadata);
|
|
const status = asOptionalRecord(pipelineRun.status);
|
|
const condition = latestCondition(status, "Succeeded");
|
|
return {
|
|
name: stringOrNull(metadata?.name),
|
|
namespace: stringOrNull(metadata?.namespace),
|
|
succeeded: pipelineRunSucceeded(pipelineRun),
|
|
reason: stringOrNull(condition?.reason),
|
|
startTime: stringOrNull(status?.startTime),
|
|
completionTime: stringOrNull(status?.completionTime),
|
|
};
|
|
}
|
|
|
|
function nativeArgoSummary(application: Record<string, unknown> | null): Record<string, unknown> | null {
|
|
if (application === null) return null;
|
|
const metadata = asOptionalRecord(application.metadata);
|
|
const status = asOptionalRecord(application.status);
|
|
const sync = asOptionalRecord(status?.sync);
|
|
const health = asOptionalRecord(status?.health);
|
|
return {
|
|
name: stringOrNull(metadata?.name),
|
|
namespace: stringOrNull(metadata?.namespace),
|
|
syncStatus: stringOrNull(sync?.status),
|
|
healthStatus: stringOrNull(health?.status),
|
|
revision: stringOrNull(sync?.revision),
|
|
ready: argoApplicationReady(application),
|
|
};
|
|
}
|
|
|
|
function nativeRuntimeSummary(runtime: NativeStatusSpec["runtime"], workloads: Record<string, unknown>[]): Record<string, unknown> | null {
|
|
if (runtime === null) return null;
|
|
return {
|
|
namespace: runtime.namespace,
|
|
ready: runtimeWorkloadsReady(runtime, workloads),
|
|
workloads: runtime.workloads.map((spec, index) => {
|
|
const workload = workloads[index] ?? null;
|
|
return {
|
|
kind: spec.kind,
|
|
name: spec.name,
|
|
ready: workloadReady(spec, workload),
|
|
sourceCommit: workloadSourceCommit(spec, workload),
|
|
};
|
|
}),
|
|
};
|
|
}
|
|
|
|
function latestCondition(status: Record<string, unknown> | null, type: string): Record<string, unknown> | null {
|
|
const conditions = Array.isArray(status?.conditions) ? status.conditions : [];
|
|
for (const condition of conditions as unknown[]) {
|
|
const record = asOptionalRecord(condition);
|
|
if (record?.type === type) return record;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function mergeFollowerStatus(
|
|
registry: BranchFollowerRegistry,
|
|
follower: FollowerSpec,
|
|
stored: Record<string, unknown>,
|
|
live: AdapterSummary | null,
|
|
liveRequested: boolean,
|
|
): Record<string, unknown> {
|
|
const storedSource = asOptionalRecord(stored.source);
|
|
const storedTarget = asOptionalRecord(stored.target);
|
|
const phase = live?.phase ?? stringOrNull(stored.phase) ?? "Observed";
|
|
const observedSha = live?.observedSha ?? stringOrNull(storedSource?.observedSha);
|
|
const targetSha = live?.targetSha ?? stringOrNull(storedTarget?.targetSha);
|
|
const lastTriggeredSha = live?.lastTriggeredSha ?? stringOrNull(stored.lastTriggeredSha);
|
|
const lastSucceededSha = live?.lastSucceededSha ?? stringOrNull(stored.lastSucceededSha);
|
|
return {
|
|
ok: live === null ? true : live.ok,
|
|
id: follower.id,
|
|
enabled: follower.enabled,
|
|
adapter: follower.adapter,
|
|
phase,
|
|
source: {
|
|
repository: follower.source.repository,
|
|
branch: follower.source.branch,
|
|
observedSha,
|
|
snapshotPrefix: follower.source.snapshotPrefix,
|
|
},
|
|
target: {
|
|
node: follower.target.node,
|
|
lane: follower.target.lane,
|
|
namespace: follower.target.namespace,
|
|
sentinel: follower.target.sentinel,
|
|
targetSha,
|
|
},
|
|
lastTriggeredSha,
|
|
lastSucceededSha,
|
|
pipelineRun: live?.pipelineRun ?? stringOrNull(stored.pipelineRun),
|
|
inFlightJob: live?.inFlightJob ?? stringOrNull(stored.inFlightJob),
|
|
budgetSource: follower.budgets,
|
|
updatedAt: stringOrNull(stored.updatedAt),
|
|
stateConfigMap: registry.controller.stateConfigMapName,
|
|
live: liveRequested,
|
|
message: live?.message ?? stringOrNull(stored.decision) ?? "no controller state yet",
|
|
warnings: Array.isArray(stored.warnings) ? stored.warnings.slice(0, 6) : [],
|
|
next: followerNextCommands(follower),
|
|
};
|
|
}
|
|
|
|
function readK8sState(registry: BranchFollowerRegistry, options: ParsedOptions): K8sStateRead {
|
|
const errors: string[] = [];
|
|
const stateResult = kubeJson(registry, options, `kubectl -n ${shQuote(registry.controller.namespace)} get configmap ${shQuote(registry.controller.stateConfigMapName)} -o json`, 10_000);
|
|
const deploymentResult = kubeJson(registry, options, `kubectl -n ${shQuote(registry.controller.namespace)} get deploy ${shQuote(registry.controller.deploymentName)} -o json`, 10_000);
|
|
const leaseResult = kubeJson(registry, options, `kubectl -n ${shQuote(registry.controller.namespace)} get lease ${shQuote(registry.controller.leaseName)} -o json`, 10_000);
|
|
const podSelector = labelSelector(registry.controller.labels);
|
|
const podsResult = kubePodList(registry, options, podSelector);
|
|
if (!stateResult.ok && !isNotFoundText(stateResult.error)) errors.push(`state configmap: ${stateResult.error}`);
|
|
if (!deploymentResult.ok && !isNotFoundText(deploymentResult.error)) errors.push(`deployment: ${deploymentResult.error}`);
|
|
if (!leaseResult.ok && !isNotFoundText(leaseResult.error)) errors.push(`lease: ${leaseResult.error}`);
|
|
if (!podsResult.ok && !isNotFoundText(podsResult.error)) errors.push(`pods: ${podsResult.error}`);
|
|
const stateByFollower: Record<string, Record<string, unknown>> = {};
|
|
const data = asOptionalRecord(stateResult.value?.data);
|
|
if (data !== null) {
|
|
for (const [key, value] of Object.entries(data)) {
|
|
if (key.startsWith("_")) continue;
|
|
if (typeof value !== "string") continue;
|
|
const parsed = parseJsonObject(value);
|
|
if (parsed !== null) stateByFollower[key] = parsed;
|
|
}
|
|
}
|
|
return {
|
|
ok: errors.length === 0,
|
|
stateByFollower,
|
|
stateConfigMapPresent: stateResult.value !== null,
|
|
deployment: deploymentResult.value,
|
|
lease: leaseResult.value,
|
|
pods: podsResult.value,
|
|
errors,
|
|
};
|
|
}
|
|
|
|
function kubeJson(registry: BranchFollowerRegistry, options: ParsedOptions, command: string, timeoutMs: number): { ok: boolean; value: Record<string, unknown> | null; error: string } {
|
|
const result = runKubeScript(registry, options, `set -eu\n${command}`, "", timeoutMs);
|
|
const value = result.exitCode === 0 ? parseJsonObject(result.stdout) : null;
|
|
return {
|
|
ok: result.exitCode === 0 && value !== null,
|
|
value,
|
|
error: redactText(tailText(result.stderr || result.stdout, 800)),
|
|
};
|
|
}
|
|
|
|
function kubePodList(registry: BranchFollowerRegistry, options: ParsedOptions, selector: string): { ok: boolean; value: Record<string, unknown> | null; error: string } {
|
|
const command = `kubectl -n ${shQuote(registry.controller.namespace)} get pods -l ${shQuote(selector)} -o name`;
|
|
const result = runKubeScript(registry, options, `set -eu\n${command}`, "", 10_000);
|
|
const names = result.stdout
|
|
.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter((line) => line.length > 0)
|
|
.map((line) => line.replace(/^pod\//u, ""));
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
value: result.exitCode === 0 ? { items: names.map((name) => ({ metadata: { name } })) } : null,
|
|
error: redactText(tailText(result.stderr || result.stdout, 800)),
|
|
};
|
|
}
|
|
|
|
function runKubeScript(registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number): CommandResult {
|
|
if (options.controller) {
|
|
return runCommand(["sh", "-lc", script], repoRoot, { input, timeoutMs });
|
|
}
|
|
return runCommand([transPath(), registry.controller.kubeRoute, "sh"], repoRoot, { input: `${script}\n`, timeoutMs });
|
|
}
|
|
|
|
function writeFollowerState(registry: BranchFollowerRegistry, state: FollowerState, options: ParsedOptions): CommandResult {
|
|
const json = JSON.stringify(state);
|
|
const dataPatch = JSON.stringify({ data: { [state.id]: json, _updatedAt: new Date().toISOString(), _specRef: SPEC_REF } });
|
|
const script = [
|
|
"set -eu",
|
|
`kubectl -n ${shQuote(registry.controller.namespace)} create configmap ${shQuote(registry.controller.stateConfigMapName)} --from-literal=_createdAt="$(date -Iseconds)" --dry-run=client -o yaml | kubectl apply -f - >/dev/null`,
|
|
`kubectl -n ${shQuote(registry.controller.namespace)} patch configmap ${shQuote(registry.controller.stateConfigMapName)} --type merge -p ${shQuote(dataPatch)} >/dev/null`,
|
|
].join("\n");
|
|
return runKubeScript(registry, options, script, "", 10_000);
|
|
}
|
|
|
|
function runControllerReconcileJob(registry: BranchFollowerRegistry, options: ParsedOptions, mode: { dryRun: boolean; wait: boolean; recordState: boolean }): Record<string, unknown> {
|
|
const timeoutSeconds = options.timeoutSeconds ?? registry.controller.budgets.runOnceSeconds;
|
|
const jobName = `${registry.controller.deploymentName}-once-${Date.now().toString(36)}`.slice(0, 63);
|
|
const manifest = renderControllerReconcileJob(registry, options, jobName, mode, timeoutSeconds);
|
|
const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
|
|
const manifestBase64 = Buffer.from(manifestYaml, "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
"tmp=$(mktemp)",
|
|
"base64 -d >\"$tmp\" <<'UNIDESK_CICD_RECONCILE_JOB_B64'",
|
|
manifestBase64,
|
|
"UNIDESK_CICD_RECONCILE_JOB_B64",
|
|
`kubectl -n ${shQuote(registry.controller.namespace)} delete job ${shQuote(jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`,
|
|
`kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`,
|
|
mode.wait ? waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds) : "true",
|
|
].join("\n");
|
|
const result = runKubeScript(registry, options, script, "", (timeoutSeconds + 20) * 1000);
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
name: jobName,
|
|
namespace: registry.controller.namespace,
|
|
mode: mode.dryRun ? "status-refresh" : "confirm-reconcile",
|
|
execution: "k8s-native-job",
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
message: result.exitCode === 0 ? "reconcile job completed" : redactText(tailText(result.stderr || result.stdout, 800)),
|
|
};
|
|
}
|
|
|
|
function renderControllerReconcileJob(registry: BranchFollowerRegistry, options: ParsedOptions, jobName: string, mode: { dryRun: boolean; recordState: boolean }, timeoutSeconds: number): Record<string, unknown> {
|
|
const labels = { ...registry.controller.labels, "app.kubernetes.io/component": "cicd-reconcile-job" };
|
|
const commandArgs = [
|
|
"bun",
|
|
"scripts/cli.ts",
|
|
"cicd",
|
|
"branch-follower",
|
|
"run-once",
|
|
...(options.followerId === null ? ["--all"] : ["--follower", options.followerId]),
|
|
mode.dryRun ? "--dry-run" : "--confirm",
|
|
"--wait",
|
|
"--controller",
|
|
"--config",
|
|
"config/cicd-branch-followers.yaml",
|
|
"--timeout-seconds",
|
|
String(timeoutSeconds),
|
|
...(mode.recordState ? ["--record-state"] : []),
|
|
];
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: { name: jobName, namespace: registry.controller.namespace, labels },
|
|
spec: {
|
|
backoffLimit: 0,
|
|
ttlSecondsAfterFinished: 600,
|
|
activeDeadlineSeconds: timeoutSeconds + 30,
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
serviceAccountName: registry.controller.serviceAccountName,
|
|
volumes: [
|
|
{ name: "registry", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } },
|
|
{ name: "git-mirror-cache", persistentVolumeClaim: { claimName: registry.controller.source.gitMirrorCachePvcName } },
|
|
{ name: "git-ssh", secret: { secretName: registry.controller.source.githubSsh.secretName, defaultMode: 0o400 } },
|
|
{ name: "work", emptyDir: {} },
|
|
],
|
|
containers: [
|
|
{
|
|
name: "reconcile",
|
|
image: registry.controller.image,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/bin/sh", "-lc"],
|
|
args: [controllerOneShotScript(commandArgs)],
|
|
env: [
|
|
{ name: "UNIDESK_CONTROLLER_SOURCE_BRANCH", value: registry.controller.source.branch },
|
|
{ name: "UNIDESK_CONTROLLER_SOURCE_REPOSITORY", value: registry.controller.source.repository },
|
|
{ name: "UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX", value: registry.controller.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", registry.controller.source.branch) },
|
|
],
|
|
volumeMounts: [
|
|
{ name: "registry", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true },
|
|
{ name: "git-mirror-cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
{ name: "work", mountPath: "/work" },
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function controllerOneShotScript(commandArgs: string[]): string {
|
|
return [
|
|
"set -eu",
|
|
"cd /work",
|
|
"rm -rf /work/unidesk",
|
|
"started_at=$(date -Iseconds)",
|
|
"echo \"branch-follower one-shot started ${started_at}\"",
|
|
"/etc/unidesk-cicd-branch-follower/sync-source.sh \"${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}\" \"${UNIDESK_CONTROLLER_SOURCE_BRANCH}\" \"${UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX}\" \"/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git\"",
|
|
"git clone --branch \"${UNIDESK_CONTROLLER_SOURCE_BRANCH}\" \"/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git\" /work/unidesk",
|
|
"cp /etc/unidesk-cicd-branch-follower/cicd-branch-followers.yaml /work/unidesk/config/cicd-branch-followers.yaml",
|
|
"cd /work/unidesk",
|
|
`${commandArgs.map(shQuote).join(" ")}`,
|
|
"echo \"branch-follower one-shot finished $(date -Iseconds)\"",
|
|
].join("\n");
|
|
}
|
|
|
|
function waitForJobShell(namespace: string, jobName: string, timeoutSeconds: number): string {
|
|
return [
|
|
`deadline=$(( $(date +%s) + ${timeoutSeconds} ))`,
|
|
"while true; do",
|
|
` job_json=$(kubectl -n ${shQuote(namespace)} get job ${shQuote(jobName)} -o json)`,
|
|
" phase=$(printf '%s' \"$job_json\" | node -e \"let s='';process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>{const j=JSON.parse(s);const c=j.status?.conditions||[];const done=c.find(x=>x.type==='Complete'&&x.status==='True');const failed=c.find(x=>x.type==='Failed'&&x.status==='True');process.stdout.write(done?'complete':failed?'failed':'running');})\")",
|
|
" if [ \"$phase\" = complete ]; then exit 0; fi",
|
|
" if [ \"$phase\" = failed ]; then exit 1; fi",
|
|
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then exit 124; fi",
|
|
" sleep 2",
|
|
"done",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderControllerManifests(registry: BranchFollowerRegistry): Record<string, unknown>[] {
|
|
const labels = registry.controller.labels;
|
|
const selector = labels;
|
|
return [
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: { name: registry.controller.namespace, labels },
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "ServiceAccount",
|
|
metadata: { name: registry.controller.serviceAccountName, namespace: registry.controller.namespace, labels },
|
|
},
|
|
{
|
|
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
kind: "Role",
|
|
metadata: { name: registry.controller.serviceAccountName, namespace: registry.controller.namespace, labels },
|
|
rules: [
|
|
{ apiGroups: [""], resources: ["configmaps", "pods", "events"], verbs: ["get", "list", "watch", "create", "update", "patch"] },
|
|
{ apiGroups: ["apps"], resources: ["deployments"], verbs: ["get", "list", "watch"] },
|
|
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "update", "patch"] },
|
|
{ apiGroups: ["coordination.k8s.io"], resources: ["leases"], verbs: ["get", "list", "watch", "create", "update", "patch"] },
|
|
],
|
|
},
|
|
{
|
|
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
kind: "RoleBinding",
|
|
metadata: { name: registry.controller.serviceAccountName, namespace: registry.controller.namespace, labels },
|
|
subjects: [{ kind: "ServiceAccount", name: registry.controller.serviceAccountName, namespace: registry.controller.namespace }],
|
|
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: registry.controller.serviceAccountName },
|
|
},
|
|
{
|
|
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
kind: "ClusterRole",
|
|
metadata: { name: registry.controller.serviceAccountName, labels },
|
|
rules: [
|
|
{ apiGroups: [""], resources: ["pods", "pods/log", "configmaps", "events"], verbs: ["get", "list", "watch"] },
|
|
{ apiGroups: [""], resources: ["pods/exec"], verbs: ["create"] },
|
|
{ apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] },
|
|
{ apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "watch"] },
|
|
{ apiGroups: ["argoproj.io"], resources: ["applications"], verbs: ["get", "list", "watch"] },
|
|
],
|
|
},
|
|
{
|
|
apiVersion: "rbac.authorization.k8s.io/v1",
|
|
kind: "ClusterRoleBinding",
|
|
metadata: { name: registry.controller.serviceAccountName, labels },
|
|
subjects: [{ kind: "ServiceAccount", name: registry.controller.serviceAccountName, namespace: registry.controller.namespace }],
|
|
roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: registry.controller.serviceAccountName },
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "ConfigMap",
|
|
metadata: { name: registry.controller.configMapName, namespace: registry.controller.namespace, labels },
|
|
data: {
|
|
"cicd-branch-followers.yaml": registry.rawText,
|
|
"sync-source.sh": nativeGitMirrorSyncShell(registry),
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "ConfigMap",
|
|
metadata: { name: registry.controller.stateConfigMapName, namespace: registry.controller.namespace, labels },
|
|
data: {
|
|
_createdAt: new Date().toISOString(),
|
|
_specRef: SPEC_REF,
|
|
_registrySha256: registry.rawSha256,
|
|
},
|
|
},
|
|
{
|
|
apiVersion: "coordination.k8s.io/v1",
|
|
kind: "Lease",
|
|
metadata: { name: registry.controller.leaseName, namespace: registry.controller.namespace, labels },
|
|
spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: Math.max(30, registry.controller.loop.reconcileTimeoutSeconds + 30) },
|
|
},
|
|
{
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: { name: registry.controller.deploymentName, namespace: registry.controller.namespace, labels },
|
|
spec: {
|
|
replicas: 1,
|
|
selector: { matchLabels: selector },
|
|
template: {
|
|
metadata: {
|
|
labels: selector,
|
|
annotations: {
|
|
"unidesk.pikapython.com/spec-ref": SPEC_REF,
|
|
"unidesk.pikapython.com/registry-sha256": registry.rawSha256,
|
|
"unidesk.pikapython.com/host-worktree-authority": "false",
|
|
},
|
|
},
|
|
spec: {
|
|
serviceAccountName: registry.controller.serviceAccountName,
|
|
terminationGracePeriodSeconds: 30,
|
|
volumes: [
|
|
{ name: "registry", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } },
|
|
{ name: "git-mirror-cache", persistentVolumeClaim: { claimName: registry.controller.source.gitMirrorCachePvcName } },
|
|
{ name: "git-ssh", secret: { secretName: registry.controller.source.githubSsh.secretName, defaultMode: 0o400 } },
|
|
{ name: "work", emptyDir: {} },
|
|
],
|
|
containers: [
|
|
{
|
|
name: "controller",
|
|
image: registry.controller.image,
|
|
imagePullPolicy: "IfNotPresent",
|
|
command: ["/bin/sh", "-lc"],
|
|
args: [controllerLoopScript()],
|
|
env: [
|
|
{ name: "UNIDESK_CICD_BRANCH_FOLLOWER_INTERVAL_SECONDS", value: String(registry.controller.loop.intervalSeconds) },
|
|
{ name: "UNIDESK_CICD_BRANCH_FOLLOWER_TIMEOUT_SECONDS", value: String(registry.controller.loop.reconcileTimeoutSeconds) },
|
|
{ name: "UNIDESK_CONTROLLER_GIT_MIRROR_READ_URL", value: registry.controller.source.gitMirrorReadUrl },
|
|
{ name: "UNIDESK_CONTROLLER_SOURCE_BRANCH", value: registry.controller.source.branch },
|
|
{ name: "UNIDESK_CONTROLLER_SOURCE_REPOSITORY", value: registry.controller.source.repository },
|
|
{ name: "UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX", value: registry.controller.source.sourceSnapshot.stageRefPrefix.replaceAll("{branch}", registry.controller.source.branch) },
|
|
],
|
|
volumeMounts: [
|
|
{ name: "registry", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true },
|
|
{ name: "git-mirror-cache", mountPath: "/cache" },
|
|
{ name: "git-ssh", mountPath: "/git-ssh", readOnly: true },
|
|
{ name: "work", mountPath: "/work" },
|
|
],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
];
|
|
}
|
|
|
|
function controllerLoopScript(): string {
|
|
return [
|
|
"set -eu",
|
|
"interval=\"${UNIDESK_CICD_BRANCH_FOLLOWER_INTERVAL_SECONDS}\"",
|
|
"timeout=\"${UNIDESK_CICD_BRANCH_FOLLOWER_TIMEOUT_SECONDS}\"",
|
|
"while true; do",
|
|
" started_at=$(date -Iseconds)",
|
|
" echo \"branch-follower loop started ${started_at}\"",
|
|
" cd /work",
|
|
" rm -rf /work/unidesk",
|
|
" /etc/unidesk-cicd-branch-follower/sync-source.sh \"${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}\" \"${UNIDESK_CONTROLLER_SOURCE_BRANCH}\" \"${UNIDESK_CONTROLLER_SOURCE_SNAPSHOT_PREFIX}\" \"/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git\"",
|
|
" git clone --branch \"${UNIDESK_CONTROLLER_SOURCE_BRANCH}\" \"/cache/${UNIDESK_CONTROLLER_SOURCE_REPOSITORY}.git\" /work/unidesk",
|
|
" cp /etc/unidesk-cicd-branch-follower/cicd-branch-followers.yaml /work/unidesk/config/cicd-branch-followers.yaml",
|
|
" cd /work/unidesk",
|
|
" bun scripts/cli.ts cicd branch-follower run-once --all --confirm --wait --controller --config config/cicd-branch-followers.yaml --timeout-seconds \"${timeout}\" || true",
|
|
" echo \"branch-follower loop finished $(date -Iseconds)\"",
|
|
" cd /work",
|
|
" sleep \"${interval}\"",
|
|
"done",
|
|
].join("\n");
|
|
}
|
|
|
|
function nativeGitMirrorSyncShell(registry: BranchFollowerRegistry): string {
|
|
const ssh = registry.controller.source.githubSsh;
|
|
return [
|
|
"#!/bin/sh",
|
|
"set -eu",
|
|
"repository=\"$1\"",
|
|
"branch=\"$2\"",
|
|
"snapshot_prefix=\"$3\"",
|
|
"repo_path=\"$4\"",
|
|
`private_key=${shQuote(`/git-ssh/${ssh.privateKeySecretKey}`)}`,
|
|
`proxy_host=${shQuote(ssh.proxyHost)}`,
|
|
`proxy_port=${shQuote(String(ssh.proxyPort))}`,
|
|
"mkdir -p \"$(dirname \"$repo_path\")\" /root/.ssh",
|
|
"cp \"$private_key\" /root/.ssh/id_rsa",
|
|
"chmod 0400 /root/.ssh/id_rsa",
|
|
"touch /root/.ssh/known_hosts",
|
|
"cat >/tmp/unidesk-cicd-github-proxy-connect.mjs <<'NODE_PROXY'",
|
|
"#!/usr/bin/env node",
|
|
"import net from 'node:net';",
|
|
"const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);",
|
|
"const proxyPort = Number.parseInt(proxyPortRaw || '', 10);",
|
|
"const targetPort = Number.parseInt(targetPortRaw || '', 10);",
|
|
"if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);",
|
|
"let settled = false;",
|
|
"let tunnel = false;",
|
|
"function finish(code) { if (settled) return; settled = true; process.exit(code); }",
|
|
"const socket = net.createConnection({ host: proxyHost, port: proxyPort });",
|
|
"let buffer = Buffer.alloc(0);",
|
|
"socket.setTimeout(15000, () => { socket.destroy(); finish(65); });",
|
|
"socket.on('connect', () => socket.write(`CONNECT ${targetHost}:${targetPort} HTTP/1.1\\r\\nHost: ${targetHost}:${targetPort}\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n`));",
|
|
"socket.on('error', () => finish(tunnel ? 69 : 66));",
|
|
"socket.on('close', () => finish(tunnel ? 0 : 68));",
|
|
"socket.on('data', function onData(chunk) {",
|
|
" buffer = Buffer.concat([buffer, chunk]);",
|
|
" const headerEnd = buffer.indexOf('\\r\\n\\r\\n');",
|
|
" if (headerEnd === -1 && buffer.length < 8192) return;",
|
|
" if (headerEnd === -1) { socket.destroy(); finish(68); return; }",
|
|
" const statusLine = buffer.slice(0, headerEnd).toString('latin1').split('\\r\\n', 1)[0] || '';",
|
|
" const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);",
|
|
" if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); finish(67); return; }",
|
|
" socket.off('data', onData);",
|
|
" socket.setTimeout(0);",
|
|
" tunnel = true;",
|
|
" const rest = buffer.slice(headerEnd + 4);",
|
|
" if (rest.length) process.stdout.write(rest);",
|
|
" process.stdin.on('error', () => {});",
|
|
" process.stdout.on('error', () => {});",
|
|
" process.stdin.pipe(socket);",
|
|
" socket.pipe(process.stdout);",
|
|
"});",
|
|
"NODE_PROXY",
|
|
"chmod 0700 /tmp/unidesk-cicd-github-proxy-connect.mjs",
|
|
"cat >/tmp/unidesk-cicd-git-ssh.sh <<SH_PROXY",
|
|
"#!/bin/sh",
|
|
"exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o \"ProxyCommand=node /tmp/unidesk-cicd-github-proxy-connect.mjs ${proxy_host} ${proxy_port} %h %p\" \"\\$@\"",
|
|
"SH_PROXY",
|
|
"chmod 0700 /tmp/unidesk-cicd-git-ssh.sh",
|
|
"export GIT_SSH=/tmp/unidesk-cicd-git-ssh.sh",
|
|
"unset GIT_SSH_COMMAND",
|
|
"remote=\"ssh://git@ssh.github.com:443/${repository}.git\"",
|
|
"if [ -d \"$repo_path/objects\" ] && [ -f \"$repo_path/HEAD\" ]; then",
|
|
" git --git-dir=\"$repo_path\" remote set-url origin \"$remote\" || git --git-dir=\"$repo_path\" remote add origin \"$remote\"",
|
|
"else",
|
|
" rm -rf \"$repo_path\"",
|
|
" git init --bare \"$repo_path\" >/dev/null",
|
|
" git --git-dir=\"$repo_path\" remote add origin \"$remote\"",
|
|
"fi",
|
|
"git --git-dir=\"$repo_path\" config uploadpack.allowReachableSHA1InWant true",
|
|
"git --git-dir=\"$repo_path\" config uploadpack.allowAnySHA1InWant true",
|
|
"timeout 30 git --git-dir=\"$repo_path\" fetch --quiet --prune origin \"+refs/heads/${branch}:refs/mirror-stage/heads/${branch}\"",
|
|
"source_sha=$(git --git-dir=\"$repo_path\" rev-parse --verify \"refs/mirror-stage/heads/${branch}^{commit}\")",
|
|
"git --git-dir=\"$repo_path\" update-ref \"refs/heads/${branch}\" \"$source_sha\"",
|
|
"if [ -n \"$snapshot_prefix\" ]; then",
|
|
" git --git-dir=\"$repo_path\" update-ref \"${snapshot_prefix%/}/$source_sha\" \"$source_sha\"",
|
|
"fi",
|
|
"git --git-dir=\"$repo_path\" update-server-info",
|
|
"printf '{\"event\":\"unidesk-cicd-git-mirror-sync\",\"repository\":\"%s\",\"branch\":\"%s\",\"commit\":\"%s\",\"sourceAuthority\":\"k8s-git-mirror-cache\"}\\n' \"$repository\" \"$branch\" \"$source_sha\"",
|
|
"",
|
|
].join("\n");
|
|
}
|
|
|
|
function selectFollowers(registry: BranchFollowerRegistry, options: ParsedOptions, opts: { includeDisabled: boolean }): FollowerSpec[] {
|
|
let selected = registry.followers;
|
|
if (options.followerId !== null) selected = selected.filter((item) => item.id === options.followerId);
|
|
else if (!options.all && options.action === "run-once") selected = selected.filter((item) => item.enabled);
|
|
if (!opts.includeDisabled) selected = selected.filter((item) => item.enabled);
|
|
if (selected.length === 0) throw new Error(options.followerId === null ? "no followers selected" : `unknown or disabled follower ${options.followerId}`);
|
|
return selected;
|
|
}
|
|
|
|
function registrySummary(registry: BranchFollowerRegistry): Record<string, unknown> {
|
|
return {
|
|
path: registry.path,
|
|
sha256: registry.rawSha256,
|
|
metadata: registry.metadata,
|
|
controller: {
|
|
namespace: registry.controller.namespace,
|
|
kubeRoute: registry.controller.kubeRoute,
|
|
deploymentName: registry.controller.deploymentName,
|
|
stateConfigMapName: registry.controller.stateConfigMapName,
|
|
leaseName: registry.controller.leaseName,
|
|
},
|
|
followers: registry.followers.map((item) => item.id),
|
|
};
|
|
}
|
|
|
|
function redactCommands(follower: FollowerSpec): Record<string, string> {
|
|
return {
|
|
plan: follower.commands.plan.argv.join(" "),
|
|
status: "native:k8s-git-mirror+tekton+argocd+runtime",
|
|
trigger: follower.commands.trigger.argv.join(" "),
|
|
events: follower.commands.events.argv.join(" "),
|
|
logs: follower.commands.logs.argv.join(" "),
|
|
};
|
|
}
|
|
|
|
function nativeStatusPlan(native: NativeStatusSpec): Record<string, unknown> {
|
|
return {
|
|
source: {
|
|
mode: "k8s-git-mirror-cache",
|
|
gitMirrorNamespace: native.source.gitMirrorNamespace,
|
|
gitMirrorDeployment: native.source.gitMirrorDeployment,
|
|
repoPath: native.source.repoPath,
|
|
},
|
|
tekton: native.tekton,
|
|
argo: native.argo,
|
|
runtime: native.runtime === null
|
|
? null
|
|
: {
|
|
namespace: native.runtime.namespace,
|
|
workloads: native.runtime.workloads.map((item) => ({ kind: item.kind, name: item.name })),
|
|
},
|
|
parsedDownstreamCliOutput: false,
|
|
};
|
|
}
|
|
|
|
function controllerStatusSummary(registry: BranchFollowerRegistry, k8s: K8sStateRead): Record<string, unknown> {
|
|
const deploymentStatus = asOptionalRecord(k8s.deployment?.status);
|
|
const available = numberOrNull(deploymentStatus?.availableReplicas) ?? 0;
|
|
const replicas = numberOrNull(deploymentStatus?.replicas) ?? 0;
|
|
const leaseSpec = asOptionalRecord(k8s.lease?.spec);
|
|
const podItems = Array.isArray(k8s.pods?.items) ? k8s.pods.items.length : null;
|
|
return {
|
|
namespace: registry.controller.namespace,
|
|
route: registry.controller.kubeRoute,
|
|
deploymentName: registry.controller.deploymentName,
|
|
deploymentPresent: k8s.deployment !== null,
|
|
availableReplicas: available,
|
|
replicas,
|
|
pods: podItems,
|
|
stateConfigMapName: registry.controller.stateConfigMapName,
|
|
stateConfigMapPresent: k8s.stateConfigMapPresent,
|
|
leaseName: registry.controller.leaseName,
|
|
leaseHolder: stringOrNull(leaseSpec?.holderIdentity),
|
|
noHostWorktreeAuthority: true,
|
|
};
|
|
}
|
|
|
|
function followerNextCommands(follower: FollowerSpec): Record<string, string> {
|
|
const next: Record<string, string> = {
|
|
status: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id}`,
|
|
liveStatus: `bun scripts/cli.ts cicd branch-follower status --follower ${follower.id} --live`,
|
|
dryRun: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --dry-run`,
|
|
trigger: `bun scripts/cli.ts cicd branch-follower run-once --follower ${follower.id} --confirm --wait`,
|
|
events: `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`,
|
|
logs: `bun scripts/cli.ts cicd branch-follower logs --follower ${follower.id}`,
|
|
};
|
|
if (follower.nativeStatus.tekton !== null) next.pipelineRuns = `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`;
|
|
if (follower.nativeStatus.argo !== null) next.argoApplication = `bun scripts/cli.ts cicd branch-follower events --follower ${follower.id}`;
|
|
return next;
|
|
}
|
|
|
|
function safeResolveString(ref: string): string | null {
|
|
try {
|
|
return resolveConfigRefString(ref, ref);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function parseJsonObject(text: string): Record<string, unknown> | null {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return null;
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
return asOptionalRecord(parsed);
|
|
} catch {
|
|
const start = trimmed.indexOf("{");
|
|
const end = trimmed.lastIndexOf("}");
|
|
if (start < 0 || end <= start) return null;
|
|
try {
|
|
const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown;
|
|
return asOptionalRecord(parsed);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
|
|
function recordAt(root: Record<string, unknown>, path: string[]): Record<string, unknown> | null {
|
|
let current: unknown = root;
|
|
for (const item of path) {
|
|
if (typeof current !== "object" || current === null || Array.isArray(current)) return null;
|
|
current = (current as Record<string, unknown>)[item];
|
|
}
|
|
return asOptionalRecord(current);
|
|
}
|
|
|
|
function asOptionalRecord(value: unknown): Record<string, unknown> | null {
|
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
|
|
return value as Record<string, unknown>;
|
|
}
|
|
|
|
function stringOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function numberOrNull(value: unknown): number | null {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
function commandCompact(result: CommandResult, options: ParsedOptions): Record<string, unknown> {
|
|
return {
|
|
argv: result.command,
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
stdoutBytes: Buffer.byteLength(result.stdout),
|
|
stderrBytes: Buffer.byteLength(result.stderr),
|
|
stdoutTail: options.full || result.exitCode !== 0 ? redactText(tailText(result.stdout, options.tailBytes)) : "",
|
|
stderrTail: options.full || result.exitCode !== 0 ? redactText(tailText(result.stderr, Math.min(options.tailBytes, 4000))) : "",
|
|
};
|
|
}
|
|
|
|
function objectRef(item: Record<string, unknown>): Record<string, string> {
|
|
const metadata = asRecord(item.metadata, "metadata");
|
|
return {
|
|
kind: stringField(item, "kind", "manifest"),
|
|
namespace: typeof metadata.namespace === "string" ? metadata.namespace : "-",
|
|
name: stringField(metadata, "name", "manifest.metadata"),
|
|
};
|
|
}
|
|
|
|
function labelSelector(labels: Record<string, string>): string {
|
|
return Object.entries(labels).map(([key, value]) => `${key}=${value}`).join(",");
|
|
}
|
|
|
|
function isNotFoundText(value: string): boolean {
|
|
return /notfound|not found|notfound|NotFound/u.test(value);
|
|
}
|
|
|
|
function shortSha(value: string | null): string {
|
|
if (value === null) return "-";
|
|
return value.length > 12 ? value.slice(0, 12) : value;
|
|
}
|
|
|
|
function safeJobSegment(value: string): string {
|
|
return value.replace(/[^A-Za-z0-9_.-]/gu, "_").slice(0, 60);
|
|
}
|
|
|
|
function tailText(text: string, maxChars: number): string {
|
|
if (text.length <= maxChars) return text;
|
|
return text.slice(text.length - maxChars);
|
|
}
|
|
|
|
function commandLabel(options: ParsedOptions): string {
|
|
return `cicd branch-follower ${options.action}`;
|
|
}
|
|
|
|
function renderResult(command: string, payload: Record<string, unknown>, options: ParsedOptions): RenderedCliResult {
|
|
const ok = payload.ok !== false;
|
|
if (options.output === "json") return renderMachine(command, payload, "json", ok);
|
|
if (options.output === "yaml") return renderMachine(command, payload, "yaml", ok);
|
|
return rendered(ok, command, renderHuman(command, payload, options));
|
|
}
|
|
|
|
function renderMachine(command: string, value: unknown, mode: "json" | "yaml", ok = true): RenderedCliResult {
|
|
return rendered(ok, command, mode === "json" ? `${JSON.stringify(value, null, 2)}\n` : `${Bun.YAML.stringify(value)}\n`, mode === "json" ? "application/json" : "application/yaml");
|
|
}
|
|
|
|
function rendered(ok: boolean, command: string, renderedText: string, contentType: RenderedCliResult["contentType"] = "text/plain"): RenderedCliResult {
|
|
return { ok, command, renderedText, contentType };
|
|
}
|
|
|
|
function renderHuman(command: string, payload: Record<string, unknown>, options: ParsedOptions): string {
|
|
if (command.endsWith(" plan")) return renderPlanHuman(payload);
|
|
if (command.endsWith(" apply")) return renderApplyHuman(payload);
|
|
if (command.endsWith(" status")) return renderStatusHuman(payload, options);
|
|
if (command.endsWith(" run-once")) return renderRunOnceHuman(payload);
|
|
if (command.endsWith(" events") || command.endsWith(" logs")) return renderDrillDownHuman(payload);
|
|
return `${JSON.stringify(payload, null, 2)}\n`;
|
|
}
|
|
|
|
function renderPlanHuman(payload: Record<string, unknown>): string {
|
|
const followers = arrayRecords(payload.followers);
|
|
const rows = followers.map((item) => {
|
|
const source = asOptionalRecord(item.source);
|
|
const target = asOptionalRecord(item.target);
|
|
const budgets = asOptionalRecord(item.budgets);
|
|
return [
|
|
item.id,
|
|
item.enabled,
|
|
item.adapter,
|
|
`${source?.repository ?? "-"}@${source?.branch ?? "-"}`,
|
|
`${target?.node ?? "-"}/${target?.lane ?? "-"}`,
|
|
budgets?.endToEndSeconds ?? "-",
|
|
arrayRecords(item.configRefGraph).length,
|
|
arrayText(item.closeoutChecks),
|
|
];
|
|
});
|
|
const next = asOptionalRecord(payload.next);
|
|
return [
|
|
`CI/CD BRANCH-FOLLOWER PLAN (${payload.ok === false ? "blocked" : "ok"})`,
|
|
"",
|
|
table(["FOLLOWER", "ENABLED", "ADAPTER", "SOURCE", "TARGET", "BUDGET", "REFS", "CHECKS"], rows),
|
|
"",
|
|
"SOURCE AUTHORITY",
|
|
`hostWorktreeAuthority=${payload.hostWorktreeAuthority === true ? "true" : "false"} mode=${asOptionalRecord(payload.sourceAuthority)?.mode ?? "-"} resolver=${asOptionalRecord(payload.sourceAuthority)?.resolver ?? "-"}`,
|
|
"",
|
|
"NEXT",
|
|
`apply: ${next?.apply ?? "-"}`,
|
|
`status: ${next?.status ?? "-"}`,
|
|
`dry-run: ${next?.dryRun ?? "-"}`,
|
|
"",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderApplyHuman(payload: Record<string, unknown>): string {
|
|
const controller = asOptionalRecord(payload.controller);
|
|
const command = asOptionalRecord(payload.command);
|
|
const next = asOptionalRecord(payload.next);
|
|
return [
|
|
`CI/CD BRANCH-FOLLOWER APPLY (${payload.ok === false ? "failed" : payload.dryRun === true ? "dry-run" : "ok"})`,
|
|
"",
|
|
table(
|
|
["NAMESPACE", "ROUTE", "DEPLOYMENT", "STATE_CM", "LEASE", "HOST_WORKTREE"],
|
|
[[controller?.namespace ?? "-", controller?.route ?? "-", controller?.deploymentName ?? "-", controller?.stateConfigMapName ?? "-", controller?.leaseName ?? "-", controller?.hostWorktreeMounted === true ? "mounted" : "not-mounted"]],
|
|
),
|
|
"",
|
|
table(["OBJECTS", "MANIFEST_SHA", "EXIT", "TIMED_OUT"], [[arrayRecords(payload.objects).length, shortSha(stringOrNull(payload.manifestSha256)), command?.exitCode ?? "-", command?.timedOut ?? "-"]]),
|
|
command?.stderrTail ? `\nSTDERR\n${command.stderrTail}` : "",
|
|
"",
|
|
"NEXT",
|
|
`status: ${next?.status ?? "-"}`,
|
|
`dry-run: ${next?.dryRun ?? "-"}`,
|
|
"",
|
|
].filter((line) => line !== "").join("\n");
|
|
}
|
|
|
|
function renderStatusHuman(payload: Record<string, unknown>, _options: ParsedOptions): string {
|
|
const controller = asOptionalRecord(payload.controller);
|
|
const followers = arrayRecords(payload.followers);
|
|
const rows = followers.map((item) => {
|
|
const source = asOptionalRecord(item.source);
|
|
const target = asOptionalRecord(item.target);
|
|
const budgets = asOptionalRecord(item.budgetSource);
|
|
return [
|
|
item.id,
|
|
item.phase,
|
|
item.adapter,
|
|
`${source?.branch ?? "-"}:${shortSha(stringOrNull(source?.observedSha))}`,
|
|
shortSha(stringOrNull(target?.targetSha)),
|
|
shortSha(stringOrNull(item.lastTriggeredSha)),
|
|
shortSha(stringOrNull(item.lastSucceededSha)),
|
|
item.pipelineRun ?? item.inFlightJob ?? "-",
|
|
budgets?.endToEndSeconds ?? "-",
|
|
item.message ?? "-",
|
|
];
|
|
});
|
|
const next = asOptionalRecord(payload.next);
|
|
const errors = Array.isArray(payload.errors) ? payload.errors : [];
|
|
return [
|
|
`CI/CD BRANCH-FOLLOWER STATUS (${payload.ok === false ? "degraded" : "ok"})`,
|
|
"",
|
|
table(
|
|
["CTRL_NS", "ROUTE", "DEPLOY", "READY", "PODS", "STATE_CM", "LEASE"],
|
|
[[controller?.namespace ?? "-", controller?.route ?? "-", controller?.deploymentName ?? "-", `${controller?.availableReplicas ?? 0}/${controller?.replicas ?? 0}`, controller?.pods ?? "-", controller?.stateConfigMapPresent === true ? "present" : "missing", controller?.leaseHolder ?? "-"]],
|
|
),
|
|
"",
|
|
table(["FOLLOWER", "PHASE", "ADAPTER", "OBSERVED", "TARGET", "TRIGGERED", "SUCCEEDED", "IN_FLIGHT", "BUDGET", "MESSAGE"], rows),
|
|
errors.length === 0 ? "" : `\nERRORS\n${errors.map((item) => `- ${item}`).join("\n")}`,
|
|
"",
|
|
"NEXT",
|
|
`live-status: ${next?.liveStatus ?? "-"}`,
|
|
`dry-run: ${next?.dryRun ?? "-"}`,
|
|
"",
|
|
].filter((line) => line !== "").join("\n");
|
|
}
|
|
|
|
function renderRunOnceHuman(payload: Record<string, unknown>): string {
|
|
const followers = arrayRecords(payload.followers);
|
|
const rows = followers.map((item) => {
|
|
const source = asOptionalRecord(item.source);
|
|
const target = asOptionalRecord(item.target);
|
|
return [
|
|
item.id,
|
|
item.phase,
|
|
`${source?.branch ?? "-"}:${shortSha(stringOrNull(source?.observedSha))}`,
|
|
shortSha(stringOrNull(target?.targetSha)),
|
|
shortSha(stringOrNull(item.lastTriggeredSha)),
|
|
item.inFlightJob ?? "-",
|
|
item.decision ?? "-",
|
|
];
|
|
});
|
|
const next = asOptionalRecord(payload.next);
|
|
return [
|
|
`CI/CD BRANCH-FOLLOWER RUN-ONCE (${payload.ok === false ? "blocked" : payload.dryRun === true ? "dry-run" : "ok"})`,
|
|
"",
|
|
table(["FOLLOWER", "PHASE", "OBSERVED", "TARGET", "TRIGGERED", "IN_FLIGHT", "DECISION"], rows),
|
|
"",
|
|
"NEXT",
|
|
`status: ${next?.status ?? "-"}`,
|
|
`live-status: ${next?.liveStatus ?? "-"}`,
|
|
"",
|
|
].join("\n");
|
|
}
|
|
|
|
function renderDrillDownHuman(payload: Record<string, unknown>): string {
|
|
if (payload.follower === undefined) {
|
|
const followers = arrayRecords(payload.followers);
|
|
return [
|
|
`CI/CD BRANCH-FOLLOWER ${String(payload.action ?? "drill-down").toUpperCase()}`,
|
|
"",
|
|
table(["FOLLOWER", "ADAPTER", "STATUS_AUTHORITY"], followers.map((item) => [item.id, item.adapter, item.statusAuthority ?? "k8s-native"])),
|
|
"",
|
|
].join("\n");
|
|
}
|
|
const summary = asOptionalRecord(payload.summary);
|
|
return [
|
|
`CI/CD BRANCH-FOLLOWER ${String(payload.action ?? "drill-down").toUpperCase()} (${payload.ok === false ? "failed" : "ok"})`,
|
|
"",
|
|
table(
|
|
["FOLLOWER", "ADAPTER", "AUTHORITY", "PHASE", "OBSERVED", "TARGET", "PIPELINERUN", "MESSAGE"],
|
|
[[payload.follower, payload.adapter ?? "-", payload.statusAuthority ?? "k8s-native", summary?.phase ?? "-", shortSha(stringOrNull(summary?.observedSha)), shortSha(stringOrNull(summary?.targetSha)), summary?.pipelineRun ?? "-", summary?.message ?? "-"]],
|
|
),
|
|
"",
|
|
].filter((line) => line !== "").join("\n");
|
|
}
|
|
|
|
function arrayRecords(value: unknown): Record<string, unknown>[] {
|
|
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
|
|
}
|
|
|
|
function arrayText(value: unknown): string {
|
|
return Array.isArray(value) ? value.map(String).join(",") : "-";
|
|
}
|
|
|
|
function table(headers: readonly string[], rows: readonly (readonly unknown[])[]): string {
|
|
const normalized = rows.map((row) => headers.map((_, index) => cell(row[index])));
|
|
const widths = headers.map((header, index) => Math.max(header.length, ...normalized.map((row) => row[index]?.length ?? 0)));
|
|
const format = (row: readonly string[]) => row.map((value, index) => value.padEnd(widths[index] ?? 0)).join(" ").trimEnd();
|
|
return [format(headers), format(headers.map((header) => "-".repeat(header.length))), ...normalized.map(format)].join("\n");
|
|
}
|
|
|
|
function cell(value: unknown): string {
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
const text = String(value).replace(/\s+/gu, " ");
|
|
return text.length > 96 ? `${text.slice(0, 93)}...` : text;
|
|
}
|