fix: add branch follower closeout gates

This commit is contained in:
Codex
2026-07-04 19:15:00 +00:00
parent 773e8f8045
commit f5490185db
10 changed files with 620 additions and 15 deletions
+2 -2
View File
@@ -214,8 +214,8 @@ function debugStepOption(value: string): BranchFollowerDebugStep {
}
function gateOption(value: string): BranchFollowerGate {
if (value === "reuse-plan" || value === "ci-taskrun-plan" || value === "cd-rollout-plan" || value === "post-deploy-health" || value === "control-plane-refresh") return value;
throw new Error("--gate must be reuse-plan, ci-taskrun-plan, cd-rollout-plan, post-deploy-health, or control-plane-refresh");
if (value === "reuse-plan" || value === "ci-taskrun-plan" || value === "cd-rollout-plan" || value === "post-deploy-health" || value === "control-plane-refresh" || value === "git-mirror-flush" || value === "runtime-closeout") return value;
throw new Error("--gate must be reuse-plan, ci-taskrun-plan, cd-rollout-plan, post-deploy-health, control-plane-refresh, git-mirror-flush, or runtime-closeout");
}
function isInClusterRuntime(): boolean {
+174 -3
View File
@@ -2,24 +2,50 @@
// Responsibility: submit bounded target-side gate Jobs and return compact evidence.
import type { CommandResult } from "./command";
import { resolveAgentRunLaneTarget } from "./agentrun-lanes";
import { yamlLaneGitMirrorJobManifest } from "./agentrun/secrets";
import { nativeHwlabControlPlaneRefreshJobManifest, runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh";
import { nativeCicdScriptLoadShell } from "./cicd-native-bundle";
import { runNativeK8sJob } from "./cicd-native";
import { waitForJobShell } from "./cicd-controller-render";
import type { BranchFollowerRegistry, FollowerSpec, ParsedOptions } from "./cicd-types";
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
import { nodeRuntimeGitMirrorJobManifest } from "./hwlab-node/render";
import { nodeRuntimeGitMirrorTarget } from "./hwlab-node/web-probe";
import { shQuote, redactText } from "./platform-infra-ops-library";
type KubeScriptRunner = (registry: BranchFollowerRegistry, options: ParsedOptions, script: string, input: string, timeoutMs: number) => CommandResult;
export async function runBranchFollowerGate(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner): Promise<Record<string, unknown>> {
if (options.gate === null) throw new Error("gate requires --gate <reuse-plan|ci-taskrun-plan|cd-rollout-plan|post-deploy-health|control-plane-refresh>");
if (options.gate === null) throw new Error("gate requires --gate <reuse-plan|ci-taskrun-plan|cd-rollout-plan|post-deploy-health|control-plane-refresh|git-mirror-flush|runtime-closeout>");
if (options.gate === "control-plane-refresh") {
return options.inCluster
? runControlPlaneRefreshGate(registry, follower, options)
: runTargetControlPlaneRefreshGateJob(registry, follower, options, runKubeScript);
}
if (options.gate === "git-mirror-flush") {
return options.inCluster
? runGitMirrorFlushGate(registry, follower, options)
: runTargetGitMirrorFlushGateJob(registry, follower, options, runKubeScript);
}
if (options.gate === "runtime-closeout" && !options.confirm) {
const timeoutSeconds = gateTimeoutSeconds(follower, options);
const jobName = `bf-gate-${safeName(follower.id)}-${safeName(options.gate)}-${Date.now().toString(36)}`.slice(0, 63);
return {
ok: true,
action: "gate",
gate: options.gate,
follower: follower.id,
dryRun: true,
sourceCommit: options.sourceCommit,
target: { name: jobName, namespace: registry.controller.namespace, execution: "k8s-native-gate-job" },
timeoutSeconds,
message: "add --confirm to run the native runtime closeout gate",
writesState: false,
parsedDownstreamCliOutput: false,
};
}
if (options.inCluster) return { ok: false, action: "gate", gate: options.gate, follower: follower.id, degradedReason: "operator-entry-required" };
const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.statusSeconds;
const timeoutSeconds = gateTimeoutSeconds(follower, options);
const jobName = `bf-gate-${safeName(follower.id)}-${safeName(options.gate)}-${Date.now().toString(36)}`.slice(0, 63);
const manifest = gateJobManifest(registry, follower, options, jobName, timeoutSeconds);
const manifestYaml = `${Bun.YAML.stringify(manifest).trim()}\n`;
@@ -56,6 +82,149 @@ export async function runBranchFollowerGate(registry: BranchFollowerRegistry, fo
};
}
function gateTimeoutSeconds(follower: FollowerSpec, options: ParsedOptions): number {
return options.timeoutSeconds ?? (options.gate === "runtime-closeout" ? follower.budgets.endToEndSeconds : follower.budgets.statusSeconds);
}
function runTargetGitMirrorFlushGateJob(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner): Record<string, unknown> {
const prepared = prepareGitMirrorFlushGate(follower, options);
if (prepared.ok !== true) return prepared;
const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.sourceSyncSeconds;
if (!options.confirm) return gitMirrorFlushDryRun(follower, options, prepared.namespace, prepared.jobName);
const manifestYaml = `${Bun.YAML.stringify(prepared.manifest).trim()}\n`;
const script = [
"set -eu",
"tmp=$(mktemp)",
"base64 -d >\"$tmp\" <<'UNIDESK_GIT_MIRROR_FLUSH_GATE_JOB'",
Buffer.from(manifestYaml, "utf8").toString("base64"),
"UNIDESK_GIT_MIRROR_FLUSH_GATE_JOB",
`kubectl -n ${shQuote(prepared.namespace)} delete job ${shQuote(prepared.jobName)} --ignore-not-found=true >/dev/null 2>&1 || true`,
`kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`,
waitForJobShell(prepared.namespace, prepared.jobName, timeoutSeconds),
].join("\n");
const startedAt = Date.now();
const command = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000);
const parsed = parseLastJsonObject(command.stdout);
const ok = command.exitCode === 0 && parsed !== null && parsed.pendingFlush !== true;
return {
ok,
action: "gate",
gate: options.gate,
follower: follower.id,
dryRun: false,
sourceCommit: options.sourceCommit,
target: { name: prepared.jobName, namespace: prepared.namespace, execution: "k8s-native-git-mirror-flush" },
result: parsed,
writesState: false,
command: {
exitCode: command.exitCode,
timedOut: command.timedOut,
elapsedMs: Date.now() - startedAt,
parseError: parsed === null ? "stdout-json-parse-failed" : null,
stdoutTail: ok ? "" : redactText(tailText(command.stdout, 1600)),
stderrTail: ok ? "" : redactText(tailText(command.stderr, 1200)),
},
parsedDownstreamCliOutput: false,
next: {
statusRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${follower.id} --step status-read --json`,
job: `bun scripts/cli.ts cicd branch-follower job --follower ${follower.id} --source-commit ${options.sourceCommit} --job git-mirror-flush --json`,
},
};
}
function runGitMirrorFlushGate(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Record<string, unknown> {
const prepared = prepareGitMirrorFlushGate(follower, options);
if (prepared.ok !== true) return prepared;
const timeoutSeconds = options.timeoutSeconds ?? follower.budgets.sourceSyncSeconds;
if (!options.confirm) return gitMirrorFlushDryRun(follower, options, prepared.namespace, prepared.jobName);
const startedAt = Date.now();
const result = runNativeK8sJob(prepared.namespace, prepared.jobName, prepared.manifest, timeoutSeconds, "flush", registry.controller.budgets);
const summary = result.summary;
const ok = result.ok && summary?.pendingFlush !== true;
return {
ok,
action: "gate",
gate: options.gate,
follower: follower.id,
dryRun: false,
sourceCommit: options.sourceCommit,
target: { name: prepared.jobName, namespace: prepared.namespace, execution: "k8s-native-git-mirror-flush" },
result: {
ok: result.ok,
completed: result.completed,
failed: result.failed,
timedOut: result.timedOut,
created: result.created,
reused: result.reused,
polls: result.polls,
elapsedMs: result.elapsedMs,
summary,
conditionReason: result.conditionReason,
conditionMessage: result.conditionMessage,
statusAuthority: result.statusAuthority,
parsedDownstreamCliOutput: false,
},
writesState: false,
command: {
elapsedMs: Date.now() - startedAt,
timeoutSeconds,
},
parsedDownstreamCliOutput: false,
next: {
statusRead: `bun scripts/cli.ts cicd branch-follower debug-step --follower ${follower.id} --step status-read --json`,
job: `bun scripts/cli.ts cicd branch-follower job --follower ${follower.id} --source-commit ${options.sourceCommit} --job git-mirror-flush --json`,
},
};
}
function prepareGitMirrorFlushGate(follower: FollowerSpec, options: ParsedOptions): { ok: true; namespace: string; jobName: string; manifest: Record<string, unknown> } | Record<string, unknown> {
if (options.sourceCommit === null) {
return {
ok: false,
action: "gate",
gate: options.gate,
follower: follower.id,
degradedReason: "source-commit-required",
message: "git-mirror-flush gate requires --source-commit <sha>",
parsedDownstreamCliOutput: false,
};
}
const jobName = nativeCapabilityJobName(follower.id, "flush", options.sourceCommit);
if (follower.adapter === "hwlab-node-runtime") {
const spec = hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node);
const mirror = nodeRuntimeGitMirrorTarget(spec);
return { ok: true, namespace: mirror.namespace, jobName, manifest: nodeRuntimeGitMirrorJobManifest(mirror, "flush", jobName) };
}
if (follower.adapter === "agentrun-yaml-lane") {
const { spec } = resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane });
return { ok: true, namespace: spec.gitMirror.namespace, jobName, manifest: yamlLaneGitMirrorJobManifest(spec, "flush", jobName) };
}
return {
ok: false,
action: "gate",
gate: options.gate,
follower: follower.id,
degradedReason: "unsupported-follower-adapter",
message: "git-mirror-flush gate is only available for followers with a native git-mirror stage",
parsedDownstreamCliOutput: false,
};
}
function gitMirrorFlushDryRun(follower: FollowerSpec, options: ParsedOptions, namespace: string, jobName: string): Record<string, unknown> {
return {
ok: true,
action: "gate",
gate: options.gate,
follower: follower.id,
dryRun: true,
sourceCommit: options.sourceCommit,
target: { name: jobName, namespace, execution: "k8s-native-git-mirror-flush" },
message: "add --confirm to run the native git-mirror flush gate",
writesState: false,
parsedDownstreamCliOutput: false,
};
}
function runTargetControlPlaneRefreshGateJob(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, runKubeScript: KubeScriptRunner): Record<string, unknown> {
if (follower.adapter !== "hwlab-node-runtime" || options.sourceCommit === null || !options.confirm) {
return runControlPlaneRefreshGate(registry, follower, options);
@@ -163,7 +332,8 @@ function runControlPlaneRefreshGate(registry: BranchFollowerRegistry, follower:
function gateJobManifest(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions, jobName: string, timeoutSeconds: number): Record<string, unknown> {
const labels = { ...registry.controller.labels, "app.kubernetes.io/component": "cicd-gate-job" };
const agentrun = follower.adapter === "agentrun-yaml-lane" ? resolveAgentRunLaneTarget({ node: follower.target.node, lane: follower.target.lane }).spec : null;
const gitopsBranch = agentrun?.gitops.branch ?? "";
const hwlab = follower.adapter === "hwlab-node-runtime" ? hwlabRuntimeLaneSpecForNode(follower.target.lane, follower.target.node) : null;
const gitopsBranch = agentrun?.gitops.branch ?? hwlab?.gitopsBranch ?? "";
const healthUrl = gateHealthUrl(follower);
const workloads = (follower.nativeStatus.runtime?.workloads ?? []).map((item) => ({ kind: item.kind, name: item.name, sourceCommit: item.sourceCommit }));
const gatePolicy = gatePolicyEnv(follower);
@@ -218,6 +388,7 @@ function gateJobManifest(registry: BranchFollowerRegistry, follower: FollowerSpe
{ name: "HEALTH_URL", value: healthUrl },
{ name: "SLOW_TASK_SECONDS", value: String(gatePolicy.slowTaskSeconds) },
{ name: "HEALTH_TIMEOUT_MS", value: String(gatePolicy.healthTimeoutMs) },
{ name: "GATE_TIMEOUT_MS", value: String(timeoutSeconds * 1000) },
{ name: "UNIDESK_CONTROLLER_GITHUB_SSH_PRIVATE_KEY", value: `/git-ssh/${registry.controller.source.githubSsh.privateKeySecretKey}` },
{ name: "UNIDESK_CONTROLLER_GITHUB_PROXY_HOST", value: registry.controller.source.githubSsh.proxyHost },
{ name: "UNIDESK_CONTROLLER_GITHUB_PROXY_PORT", value: String(registry.controller.source.githubSsh.proxyPort) },
+2
View File
@@ -21,6 +21,8 @@ export function buildCicdHelp(configPath: string, spec: string): unknown {
"bun scripts/cli.ts cicd branch-follower taskrun --follower hwlab-jd01-v03 --taskrun runtime-ready --logs-tail 120 --json",
"bun scripts/cli.ts cicd branch-follower job --follower agentrun-jd01-v02 --source-commit <sha> --job image-build --json",
"bun scripts/cli.ts cicd branch-follower gate --follower hwlab-jd01-v03 --gate control-plane-refresh --source-commit <sha> --confirm --json",
"bun scripts/cli.ts cicd branch-follower gate --follower hwlab-jd01-v03 --gate git-mirror-flush --source-commit <sha> --confirm --json",
"bun scripts/cli.ts cicd branch-follower gate --follower hwlab-jd01-v03 --gate runtime-closeout --source-commit <sha> --confirm --json",
"bun scripts/cli.ts cicd branch-follower gate --follower agentrun-jd01-v02 --gate reuse-plan --source-commit <sha> --json",
],
config: configPath,
@@ -173,6 +173,7 @@ function missingPolicyPayload(action: string, follower: FollowerSpec, registry:
}
function resolveJobTarget(registry: BranchFollowerRegistry, follower: FollowerSpec, query: string, sourceCommit: string | null): { namespace: string; jobName: string; stage: string } | null {
if (query.startsWith("bf-gate-")) return { namespace: registry.controller.namespace, jobName: query, stage: "controller-gate-job" };
if (!isStageAlias(query)) return { namespace: follower.target.namespace, jobName: query, stage: "explicit-job" };
if (sourceCommit === null) return null;
if (query === "git-mirror-sync" || query === "git-mirror-flush") {
+2
View File
@@ -66,6 +66,7 @@ export function nativeArgoSummary(application: Record<string, unknown> | null):
const sync = asOptionalRecord(status?.sync);
const health = asOptionalRecord(status?.health);
const operationState = asOptionalRecord(status?.operationState);
const syncResultResources = Array.isArray(operationState?.syncResultResources) ? operationState.syncResultResources.slice(0, 5) : [];
return {
name: stringOrNull(metadata?.name),
namespace: stringOrNull(metadata?.namespace),
@@ -80,6 +81,7 @@ export function nativeArgoSummary(application: Record<string, unknown> | null):
operationDurationSeconds: numberOrNull(operationState?.durationSeconds),
conditions: Array.isArray(status?.conditions) ? status.conditions.slice(0, 5) : [],
nonReadyResources: Array.isArray(status?.nonReadyResources) ? status.nonReadyResources.slice(0, 5) : [],
syncResultResources,
ready: argoApplicationReady(application),
};
}
+1 -1
View File
@@ -4,7 +4,7 @@
export type OutputMode = "human" | "json" | "yaml";
export type BranchFollowerAction = "help" | "plan" | "apply" | "status" | "run-once" | "debug-step" | "cleanup-state" | "events" | "logs" | "taskrun" | "job" | "runtime" | "gate";
export type BranchFollowerDebugStep = "state-read" | "controller-source" | "status-read" | "decide" | "state-write";
export type BranchFollowerGate = "reuse-plan" | "ci-taskrun-plan" | "cd-rollout-plan" | "post-deploy-health" | "control-plane-refresh";
export type BranchFollowerGate = "reuse-plan" | "ci-taskrun-plan" | "cd-rollout-plan" | "post-deploy-health" | "control-plane-refresh" | "git-mirror-flush" | "runtime-closeout";
export type BranchFollowerPhase =
| "Observed"
| "Noop"
+3 -3
View File
@@ -612,6 +612,9 @@ export async function runNodeDelegatedDomain(config: Config, domain: DelegatedNo
const result = nodeRuntimeControlPlanePlan(scoped);
return nodeScopedFullOutput(scoped) ? result : withNodeRuntimeControlPlanePlanRendered(result, scoped);
}
if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") {
return runNodeEndpointBridge(scoped);
}
if (domain === "control-plane" && scoped.node !== defaultSpec.nodeId) {
if (scoped.action === "status") {
const result = nodeRuntimeControlPlaneStatus(scoped);
@@ -637,9 +640,6 @@ export async function runNodeDelegatedDomain(config: Config, domain: DelegatedNo
}
return nodeRuntimeUnsupportedAction(scoped);
}
if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") {
return runNodeEndpointBridge(scoped);
}
if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) {
return startNodeDelegatedJob(scoped);
}