fix: add branch follower closeout gates
This commit is contained in:
Codex
+174
-3
@@ -2,24 +2,50 @@
|
||||
// Responsibility: submit bounded target-side gate Jobs and return compact evidence.
|
||||
import type { CommandResult } from "./command";
|
||||
import { resolveAgentRunLaneTarget } from "./agentrun-lanes";
|
||||
import { yamlLaneGitMirrorJobManifest } from "./agentrun/secrets";
|
||||
import { nativeHwlabControlPlaneRefreshJobManifest, runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh";
|
||||
import { nativeCicdScriptLoadShell } from "./cicd-native-bundle";
|
||||
import { runNativeK8sJob } from "./cicd-native";
|
||||
import { waitForJobShell } from "./cicd-controller-render";
|
||||
import type { BranchFollowerRegistry, FollowerSpec, ParsedOptions } from "./cicd-types";
|
||||
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
||||
import { nodeRuntimeGitMirrorJobManifest } from "./hwlab-node/render";
|
||||
import { nodeRuntimeGitMirrorTarget } from "./hwlab-node/web-probe";
|
||||
import { shQuote, redactText } from "./platform-infra-ops-library";
|
||||
|
||||
type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult;
|
||||
|
||||
export async function runBranchFollowerGate(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner): Promise<Record<string, unknown>> {
|
||||
if (options.gate === null) throw new Error("gate requires --gate <reuse-plan|ci-taskrun-plan|cd-rollout-plan|post-deploy-health|control-plane-refresh>");
|
||||
if (options.gate === null) throw new Error("gate requires --gate <reuse-plan|ci-taskrun-plan|cd-rollout-plan|post-deploy-health|control-plane-refresh|git-mirror-flush|runtime-closeout>");
|
||||
if (options.gate === "control-plane-refresh") {
|
||||
return options.inCluster
|
||||
? runControlPlaneRefreshGate(registry, follower, options)
|
||||
: runTargetControlPlaneRefreshGateJob(registry, follower, options, runKubeScript);
|
||||
}
|
||||
if (options.gate === "git-mirror-flush") {
|
||||
return options.inCluster
|
||||
? runGitMirrorFlushGate(registry, follower, options)
|
||||
: runTargetGitMirrorFlushGateJob(registry, follower, options, runKubeScript);
|
||||
}
|
||||
if (options.gate === "runtime-closeout" && !options.confirm) {
|
||||
const timeoutSeconds = gateTimeoutSeconds(follower, options);
|
||||
const jobName = `bf-gate-${safeName(follower.id)}-${safeName(options.gate)}-${Date.now().toString(36)}`.slice(0, 63);
|
||||
return {
|
||||
ok: true,
|
||||
action: "gate",
|
||||
gate: options.gate,
|
||||
follower: follower.id,
|
||||
dryRun: true,
|
||||
sourceCommit: options.sourceCommit,
|
||||
target: { name: jobName, namespace: registry.controller.namespace, execution: "k8s-native-gate-job" },
|
||||
timeoutSeconds,
|
||||
message: "add --confirm to run the native runtime closeout gate",
|
||||
writesState: false,
|
||||
parsedDownstreamCliOutput: false,
|
||||
};
|
||||
}
|
||||
if (options.inCluster) return { ok: false, action: "gate", gate: options.gate, follower: follower.id, degradedReason: "operator-entry-required" };
|
||||
const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.statusSeconds;
|
||||
const timeoutSeconds = gateTimeoutSeconds(follower, options);
|
||||
const jobName = `bf-gate-${safeName(follower.id)}-${safeName(options.gate)}-${Date.now().toString(36)}`.slice(0, 63);
|
||||
const manifest = gateJobManifest(registry, follower, options, jobName, timeoutSeconds);
|
||||
const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
|
||||
@@ -56,6 +82,149 @@ export async function runBranchFollowerGate(registry: BranchFollowerRegistry, fo
|
||||
};
|
||||
}
|
||||
|
||||
function gateTimeoutSeconds(follower: FollowerSpec, options: ParsedOptions): number {
|
||||
return options.timeoutSeconds ?? (options.gate === "runtime-closeout" ? follower.budgets.endToEndSeconds : follower.budgets.statusSeconds);
|
||||
}
|
||||
|
||||
function runTargetGitMirrorFlushGateJob(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner): Record<string, unknown> {
|
||||
const prepared = prepareGitMirrorFlushGate(follower, options);
|
||||
if (prepared.ok !== true) return prepared;
|
||||
const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.sourceSyncSeconds;
|
||||
if (!options.confirm) return gitMirrorFlushDryRun(follower, options, prepared.namespace, prepared.jobName);
|
||||
const manifestYaml = `${Bun.YAML.stringify(prepared.manifest).trim()}\n`;
|
||||
const script = [
|
||||
"set -eu",
|
||||
"tmp=$(mktemp)",
|
||||
"base64 -d >\"$tmp\" <<'UNIDESK_GIT_MIRROR_FLUSH_GATE_JOB'",
|
||||
Buffer.from(manifestYaml, "utf8").toString("base64"),
|
||||
"UNIDESK_GIT_MIRROR_FLUSH_GATE_JOB",
|
||||
`kubectl -n ${shQuote(prepared.namespace)} delete job ${shQuote(prepared.jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`,
|
||||
`kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`,
|
||||
waitForJobShell(prepared.namespace, prepared.jobName, timeoutSeconds),
|
||||
].join("\n");
|
||||
const startedAt = Date.now();
|
||||
const command = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000);
|
||||
const parsed = parseLastJsonObject(command.stdout);
|
||||
const ok = command.exitCode === 0 && parsed !== null && parsed.pendingFlush !== true;
|
||||
return {
|
||||
ok,
|
||||
action: "gate",
|
||||
gate: options.gate,
|
||||
follower: follower.id,
|
||||
dryRun: false,
|
||||
sourceCommit: options.sourceCommit,
|
||||
target: { name: prepared.jobName, namespace: prepared.namespace, execution: "k8s-native-git-mirror-flush" },
|
||||
result: parsed,
|
||||
writesState: false,
|
||||
command: {
|
||||
exitCode: command.exitCode,
|
||||
timedOut: command.timedOut,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
parseError: parsed === null ? "stdout-json-parse-failed" : null,
|
||||
stdoutTail: ok ? "" : redactText(tailText(command.stdout, 1600)),
|
||||
stderrTail: ok ? "" : redactText(tailText(command.stderr, 1200)),
|
||||
},
|
||||
parsedDownstreamCliOutput: false,
|
||||
next: {
|
||||
statusRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${follower.id} --step status-read --json`,
|
||||
job: `bun scripts/cli.ts cicd branch-follower job --follower ${follower.id} --source-commit ${options.sourceCommit} --job git-mirror-flush --json`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function runGitMirrorFlushGate(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Record<string, unknown> {
|
||||
const prepared = prepareGitMirrorFlushGate(follower, options);
|
||||
if (prepared.ok !== true) return prepared;
|
||||
const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.sourceSyncSeconds;
|
||||
if (!options.confirm) return gitMirrorFlushDryRun(follower, options, prepared.namespace, prepared.jobName);
|
||||
const startedAt = Date.now();
|
||||
const result = runNativeK8sJob(prepared.namespace, prepared.jobName, prepared.manifest, timeoutSeconds, "flush", registry.controller.budgets);
|
||||
const summary = result.summary;
|
||||
const ok = result.ok && summary?.pendingFlush !== true;
|
||||
return {
|
||||
ok,
|
||||
action: "gate",
|
||||
gate: options.gate,
|
||||
follower: follower.id,
|
||||
dryRun: false,
|
||||
sourceCommit: options.sourceCommit,
|
||||
target: { name: prepared.jobName, namespace: prepared.namespace, execution: "k8s-native-git-mirror-flush" },
|
||||
result: {
|
||||
ok: result.ok,
|
||||
completed: result.completed,
|
||||
failed: result.failed,
|
||||
timedOut: result.timedOut,
|
||||
created: result.created,
|
||||
reused: result.reused,
|
||||
polls: result.polls,
|
||||
elapsedMs: result.elapsedMs,
|
||||
summary,
|
||||
conditionReason: result.conditionReason,
|
||||
conditionMessage: result.conditionMessage,
|
||||
statusAuthority: result.statusAuthority,
|
||||
parsedDownstreamCliOutput: false,
|
||||
},
|
||||
writesState: false,
|
||||
command: {
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
timeoutSeconds,
|
||||
},
|
||||
parsedDownstreamCliOutput: false,
|
||||
next: {
|
||||
statusRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${follower.id} --step status-read --json`,
|
||||
job: `bun scripts/cli.ts cicd branch-follower job --follower ${follower.id} --source-commit ${options.sourceCommit} --job git-mirror-flush --json`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function prepareGitMirrorFlushGate(follower: FollowerSpec, options: ParsedOptions): { ok: true; namespace: string; jobName: string; manifest: Record<string, unknown> } | Record<string, unknown> {
|
||||
if (options.sourceCommit === null) {
|
||||
return {
|
||||
ok: false,
|
||||
action: "gate",
|
||||
gate: options.gate,
|
||||
follower: follower.id,
|
||||
degradedReason: "source-commit-required",
|
||||
message: "git-mirror-flush gate requires --source-commit <sha>",
|
||||
parsedDownstreamCliOutput: false,
|
||||
};
|
||||
}
|
||||
const jobName = nativeCapabilityJobName(follower.id, "flush", options.sourceCommit);
|
||||
if (follower.adapter === "hwlab-node-runtime") {
|
||||
const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node);
|
||||
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
||||
return { ok: true, namespace: mirror.namespace, jobName, manifest: nodeRuntimeGitMirrorJobManifest(mirror, "flush", jobName) };
|
||||
}
|
||||
if (follower.adapter === "agentrun-yaml-lane") {
|
||||
const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane });
|
||||
return { ok: true, namespace: spec.gitMirror.namespace, jobName, manifest: yamlLaneGitMirrorJobManifest(spec, "flush", jobName) };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
action: "gate",
|
||||
gate: options.gate,
|
||||
follower: follower.id,
|
||||
degradedReason: "unsupported-follower-adapter",
|
||||
message: "git-mirror-flush gate is only available for followers with a native git-mirror stage",
|
||||
parsedDownstreamCliOutput: false,
|
||||
};
|
||||
}
|
||||
|
||||
function gitMirrorFlushDryRun(follower: FollowerSpec, options: ParsedOptions, namespace: string, jobName: string): Record<string, unknown> {
|
||||
return {
|
||||
ok: true,
|
||||
action: "gate",
|
||||
gate: options.gate,
|
||||
follower: follower.id,
|
||||
dryRun: true,
|
||||
sourceCommit: options.sourceCommit,
|
||||
target: { name: jobName, namespace, execution: "k8s-native-git-mirror-flush" },
|
||||
message: "add --confirm to run the native git-mirror flush gate",
|
||||
writesState: false,
|
||||
parsedDownstreamCliOutput: false,
|
||||
};
|
||||
}
|
||||
|
||||
function runTargetControlPlaneRefreshGateJob(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner): Record<string, unknown> {
|
||||
if (follower.adapter !== "hwlab-node-runtime" || options.sourceCommit === null || !options.confirm) {
|
||||
return runControlPlaneRefreshGate(registry, follower, options);
|
||||
@@ -163,7 +332,8 @@ function runControlPlaneRefreshGate(registry: BranchFollowerRegistry, follower:
|
||||
function gateJobManifest(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, jobName: string, timeoutSeconds: number): Record<string, unknown> {
|
||||
const labels = { ...registry.controller.labels, "app.kubernetes.io/component": "cicd-gate-job" };
|
||||
const agentrun = follower.adapter === "agentrun-yaml-lane" ? resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }).spec : null;
|
||||
const gitopsBranch = agentrun?.gitops.branch ?? "";
|
||||
const hwlab = follower.adapter === "hwlab-node-runtime" ? hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node) : null;
|
||||
const gitopsBranch = agentrun?.gitops.branch ?? hwlab?.gitopsBranch ?? "";
|
||||
const healthUrl = gateHealthUrl(follower);
|
||||
const workloads = (follower.nativeStatus.runtime?.workloads ?? []).map((item) => ({ kind: item.kind, name: item.name, sourceCommit: item.sourceCommit }));
|
||||
const gatePolicy = gatePolicyEnv(follower);
|
||||
@@ -218,6 +388,7 @@ function gateJobManifest(registry: BranchFollowerRegistry, follower: FollowerSpe
|
||||
{ name: "HEALTH_URL", value: healthUrl },
|
||||
{ name: "SLOW_TASK_SECONDS", value: String(gatePolicy.slowTaskSeconds) },
|
||||
{ name: "HEALTH_TIMEOUT_MS", value: String(gatePolicy.healthTimeoutMs) },
|
||||
{ name: "GATE_TIMEOUT_MS", value: String(timeoutSeconds * 1000) },
|
||||
{ name: "UNIDESK_CONTROLLER_GITHUB_SSH_PRIVATE_KEY", value: `/git-ssh/${registry.controller.source.githubSsh.privateKeySecretKey}` },
|
||||
{ name: "UNIDESK_CONTROLLER_GITHUB_PROXY_HOST", value: registry.controller.source.githubSsh.proxyHost },
|
||||
{ name: "UNIDESK_CONTROLLER_GITHUB_PROXY_PORT", value: String(registry.controller.source.githubSsh.proxyPort) },
|
||||
|
||||
Reference in New Issue
Block a user