fix: add branch follower closeout gates

This commit is contained in:
Codex
2026-07-04 19:15:00 +00:00
parent 773e8f8045
commit f5490185db
10 changed files with 620 additions and 15 deletions
+251 -2
View File
@@ -24,6 +24,7 @@ const workloads = parseWorkloads(process.env.WORKLOADS_B64 || "");
const healthUrl = process.env.HEALTH_URL || "";
const slowTaskSeconds = requiredPositiveIntEnv("SLOW_TASK_SECONDS");
const healthTimeoutMs = requiredPositiveIntEnv("HEALTH_TIMEOUT_MS");
const gateTimeoutMs = optionalPositiveIntEnv("GATE_TIMEOUT_MS") ?? healthTimeoutMs;
const errors = [];
const branchCommit = rev(`refs/heads/${sourceBranch}`);
@@ -45,10 +46,11 @@ if (gate === "reuse-plan") evidence = await reusePlanEvidence(sourceCommit);
else if (gate === "ci-taskrun-plan") evidence = await ciTaskRunEvidence(sourceCommit);
else if (gate === "cd-rollout-plan") evidence = await cdRolloutEvidence(sourceCommit);
else if (gate === "post-deploy-health") evidence = await postDeployHealthEvidence(sourceCommit);
else if (gate === "runtime-closeout") evidence = await runtimeCloseoutEvidence(sourceCommit);
else fail(`unsupported gate ${gate}`);
const ok = errors.length === 0 && evidence?.ok === true;
console.log(JSON.stringify({
const payload = {
ok,
gate,
follower,
@@ -62,7 +64,9 @@ console.log(JSON.stringify({
statusAuthority: "kubernetes-api-serviceaccount",
parsedDownstreamCliOutput: false,
bounded: true,
}));
};
console.log(JSON.stringify(payload));
if (!ok) process.exit(2);
async function reusePlanEvidence(commit) {
const reuse = readReuseConfig(commit);
@@ -417,6 +421,68 @@ async function postDeployHealthEvidence(commit) {
};
}
async function runtimeCloseoutEvidence(commit) {
const startedAt = Date.now();
const refresh = await argoRefresh();
const deadline = Date.now() + gateTimeoutMs;
let polls = 0;
let latest = null;
while (Date.now() <= deadline) {
polls += 1;
latest = await runtimeCloseoutStatus(commit);
if (latest.ok === true) break;
if (Date.now() + 2000 > deadline) break;
await delay(2000);
}
return {
ok: latest?.ok === true,
sourceCommit: shortSha(commit),
gitMirror: compactGitMirrorEvidence(gitMirror),
refresh,
closeout: latest,
polls,
elapsedMs: Date.now() - startedAt,
writesState: false,
contract: {
gate: "argo-runtime-only",
excludes: ["git-mirror-flush", "state-write"],
expectation: "Argo refresh and runtime readiness are independently triggerable; git-mirror post-flush is validated by the git-mirror-flush gate.",
},
};
}
async function runtimeCloseoutStatus(commit) {
const pipelineRunName = commit && pipelineRunPrefix ? `${pipelineRunPrefix}-${commit.slice(0, 12)}` : "";
const pipelineRun = pipelineRunName && tektonNamespace
? pipelineRunStatus(await getJson(`/apis/tekton.dev/v1/namespaces/${encodeURIComponent(tektonNamespace)}/pipelineruns/${encodeURIComponent(pipelineRunName)}`, false))
: { present: null, succeeded: null, reason: "tekton-not-configured" };
const argo = await argoSummary();
const runtime = await runtimeSummary(commit);
return {
ok: source.snapshotReady === true && pipelineRun.succeeded === true && argo.ready === true && runtime.ready === true && runtime.aligned === true,
source,
pipelineRun,
argo,
runtime,
};
}
async function argoRefresh() {
if (!argoNamespace || !argoApplication) return { ok: true, skipped: true, reason: "argo-not-configured" };
const path = `/apis/argoproj.io/v1alpha1/namespaces/${encodeURIComponent(argoNamespace)}/applications/${encodeURIComponent(argoApplication)}`;
const patched = await patchJson(path, { metadata: { annotations: { "argocd.argoproj.io/refresh": "hard" } } });
return {
ok: patched !== null,
namespace: argoNamespace,
application: argoApplication,
refresh: "hard",
resourceVersion: str(patched?.metadata?.resourceVersion),
statusAuthority: "kubernetes-api-serviceaccount",
parsedDownstreamCliOutput: false,
valuesRedacted: true,
};
}
function readReuseConfig(commit) {
if (!commit || !sourceStageRef) return { present: false, reason: "source-commit-missing" };
try {
@@ -517,6 +583,7 @@ async function argoSummary() {
const sync = app?.status?.sync || {};
const health = app?.status?.health || {};
const op = app?.status?.operationState || {};
const problemWorkloads = await argoProblemWorkloadSummaries(app);
return {
name: argoApplication,
namespace: argoNamespace,
@@ -525,10 +592,141 @@ async function argoSummary() {
revision: str(sync.revision),
operationPhase: str(op.phase),
operationMessage: str(op.message),
conditions: compactArgoConditions(app?.status?.conditions),
nonReadyResources: compactArgoNonReadyResources(app?.status?.resources),
syncResultResources: compactArgoSyncResultResources(op?.syncResult?.resources),
problemWorkloads,
ready: sync.status === "Synced" && health.status === "Healthy",
};
}
async function argoProblemWorkloadSummaries(app) {
const resources = Array.isArray(app?.status?.resources) ? app.status.resources : [];
const syncResources = Array.isArray(app?.status?.operationState?.syncResult?.resources) ? app.status.operationState.syncResult.resources : [];
const names = uniqueStrings([
...resources
.filter((item) => item?.kind === "Deployment" && (item.status === "OutOfSync" || (item.health?.status && item.health.status !== "Healthy")))
.map((item) => str(item.name))
.filter(Boolean),
...syncResources
.filter((item) => item?.kind === "Deployment" && problemArgoSyncResource(item))
.map((item) => str(item.name))
.filter(Boolean),
]).slice(0, 8);
const rows = [];
for (const name of names) rows.push(await deploymentProblemSummary(runtimeNamespace || "default", name));
return rows;
}
async function deploymentProblemSummary(namespace, name) {
const deployment = await getJson(`/apis/apps/v1/namespaces/${encodeURIComponent(namespace)}/deployments/${encodeURIComponent(name)}`, false);
const selector = deploymentSelector(deployment);
const pods = selector ? await getJson(`/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${encodeURIComponent(selector)}`, false) : null;
const conditions = Array.isArray(deployment?.status?.conditions)
? deployment.status.conditions
.filter((item) => item?.status !== "True" || item?.type === "Progressing")
.slice(-4)
.map((item) => ({
type: str(item?.type),
status: str(item?.status),
reason: str(item?.reason),
message: shortText(str(item?.message) || ""),
}))
: [];
return {
kind: "Deployment",
namespace,
name,
desired: deployment?.spec?.replicas ?? 1,
readyReplicas: deployment?.status?.readyReplicas ?? 0,
updatedReplicas: deployment?.status?.updatedReplicas ?? 0,
unavailableReplicas: deployment?.status?.unavailableReplicas ?? null,
conditions,
pods: compactProblemPods(pods),
};
}
function deploymentSelector(deployment) {
const labels = deployment?.spec?.selector?.matchLabels;
if (!labels || typeof labels !== "object" || Array.isArray(labels)) return null;
const pairs = Object.entries(labels)
.filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string")
.map(([key, value]) => `${key}=${value}`);
return pairs.length === 0 ? null : pairs.join(",");
}
function compactProblemPods(list) {
const items = Array.isArray(list?.items) ? list.items : [];
return items.slice(0, 6).map((pod) => {
const statuses = Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : [];
return {
name: str(pod?.metadata?.name),
phase: str(pod?.status?.phase),
ready: statuses.every((item) => item?.ready === true),
containers: statuses.slice(0, 4).map((item) => ({
name: str(item?.name),
ready: item?.ready === true,
restartCount: typeof item?.restartCount === "number" ? item.restartCount : null,
waitingReason: str(item?.state?.waiting?.reason),
waitingMessage: shortText(str(item?.state?.waiting?.message) || ""),
terminatedReason: str(item?.lastState?.terminated?.reason) || str(item?.state?.terminated?.reason),
exitCode: typeof item?.lastState?.terminated?.exitCode === "number" ? item.lastState.terminated.exitCode : typeof item?.state?.terminated?.exitCode === "number" ? item.state.terminated.exitCode : null,
})),
};
});
}
function compactArgoConditions(value) {
return Array.isArray(value)
? value.slice(0, 5).map((item) => ({
type: str(item?.type),
message: shortText(str(item?.message) || ""),
lastTransitionTime: str(item?.lastTransitionTime),
}))
: [];
}
function compactArgoNonReadyResources(value) {
return Array.isArray(value)
? value
.filter((item) => item?.health?.status && item.health.status !== "Healthy")
.slice(0, 5)
.map((item) => ({
kind: str(item?.kind),
namespace: str(item?.namespace),
name: str(item?.name),
status: str(item?.status),
healthStatus: str(item?.health?.status),
healthMessage: shortText(str(item?.health?.message) || ""),
}))
: [];
}
function compactArgoSyncResultResources(value) {
return Array.isArray(value)
? value
.filter(problemArgoSyncResource)
.slice(0, 8)
.map((item) => ({
group: str(item?.group),
kind: str(item?.kind),
namespace: str(item?.namespace),
name: str(item?.name),
status: str(item?.status),
hookPhase: str(item?.hookPhase),
syncPhase: str(item?.syncPhase),
message: shortText(str(item?.message) || ""),
}))
: [];
}
function problemArgoSyncResource(item) {
const message = String(item?.message || "");
return (item?.status && item.status !== "Synced")
|| (item?.hookPhase && item.hookPhase !== "Succeeded")
|| /fail|error|backoff|forbidden|invalid|denied|exceeded/iu.test(message);
}
async function runtimeSummary(expected) {
if (!runtimeNamespace || workloads.length === 0) return { ready: null, aligned: null, reason: "runtime-not-configured" };
const rows = [];
@@ -723,6 +921,49 @@ async function getJson(path, required) {
});
}
async function patchJson(path, value) {
const host = process.env.KUBERNETES_SERVICE_HOST;
const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443");
const tokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token";
const caPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt";
if (!host || !existsSync(tokenPath) || !existsSync(caPath)) fail("kubernetes serviceaccount is unavailable");
const token = readFileSync(tokenPath, "utf8").trim();
const ca = readFileSync(caPath);
const payload = JSON.stringify(value);
return await new Promise((resolve, reject) => {
const req = https.request({
host,
port,
path,
method: "PATCH",
ca,
headers: {
authorization: `Bearer ${token}`,
"content-type": "application/merge-patch+json",
"content-length": Buffer.byteLength(payload),
},
}, (res) => {
let body = "";
res.setEncoding("utf8");
res.on("data", (chunk) => { body += chunk; });
res.on("end", () => {
if ((res.statusCode || 0) < 200 || (res.statusCode || 0) >= 300) return reject(new Error(shortText(body || `kube api ${res.statusCode}`)));
try { resolve(JSON.parse(body)); } catch (error) { reject(error); }
});
});
req.on("error", reject);
req.write(payload);
req.end();
}).catch((error) => {
errors.push(`${path}: ${shortText(error?.message || String(error))}`);
return null;
});
}
function delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function httpProbe(url) {
const client = url.startsWith("https:") ? https : http;
const started = Date.now();
@@ -746,6 +987,14 @@ function parseWorkloads(value) {
}
}
function optionalPositiveIntEnv(name) {
const raw = process.env[name] || "";
if (!raw) return null;
const value = Number.parseInt(raw, 10);
if (!Number.isInteger(value) || value <= 0) fail(`${name} must be a positive integer`);
return value;
}
function rev(ref) {
try {
const out = execFileSync("git", [`--git-dir=${repoPath}`, "rev-parse", "--verify", `${ref}^{commit}`], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
@@ -137,6 +137,21 @@ if (key === "pipelineRun") {
};
} else if (key === "argoApplication") {
const resources = Array.isArray(input?.status?.resources) ? input.status.resources : [];
const syncResultResources = Array.isArray(input?.status?.operationState?.syncResult?.resources)
? input.status.operationState.syncResult.resources
.filter(problemSyncResource)
.slice(0, 8)
.map((item) => ({
group: item.group || null,
kind: item.kind || null,
namespace: item.namespace || null,
name: item.name || null,
status: item.status || null,
hookPhase: item.hookPhase || null,
syncPhase: item.syncPhase || null,
message: item.message || null,
}))
: [];
const nonReadyResources = resources
.filter((item) => item?.health?.status && item.health.status !== "Healthy")
.slice(0, 8)
@@ -166,6 +181,7 @@ if (key === "pipelineRun") {
startedAt: input.status.operationState.startedAt || null,
finishedAt: input.status.operationState.finishedAt || null,
durationSeconds: durationSeconds(input.status.operationState.startedAt, input.status.operationState.finishedAt),
syncResultResources,
}
: null,
},
@@ -194,3 +210,10 @@ if (key === "pipelineRun") {
}
console.log(JSON.stringify(output));
function problemSyncResource(item) {
const message = String(item?.message || "");
return (item?.status && item.status !== "Synced")
|| (item?.hookPhase && item.hookPhase !== "Succeeded")
|| /fail|error|backoff|forbidden|invalid|denied|exceeded/iu.test(message);
}
+161 -4
View File
@@ -51,10 +51,13 @@ async function main() {
const podSummaries = pods.slice(0, Math.max(1, maxContainers)).map(podSummary);
const logTargets = selectLogTargets(pods).slice(0, maxContainers);
const logs = [];
const gateResults = [];
const perContainerBytes = Math.max(1, Math.floor(maxLogBytes / Math.max(1, logTargets.length)));
for (const target of logTargets) {
const read = await readPodLog(target.podName, target.container, logsTailLines, perContainerBytes);
const text = read.tail || "";
const parsedGateResult = compactGateResult(parseLastJsonObject(text));
if (parsedGateResult !== null) gateResults.push(parsedGateResult);
logs.push({
ok: read.ok,
degradedReason: read.degradedReason,
@@ -63,16 +66,17 @@ async function main() {
container: target.container,
lineCount: text.length === 0 ? 0 : text.split(/\r?\n/u).filter((line) => line.length > 0).length,
bytes: Buffer.byteLength(text, "utf8"),
tail: text,
tail: compactLogTail(text, parsedGateResult),
nodeCicdTiming: lastNodeCicdTiming(text),
});
}
const logFailures = logs.filter((item) => item.ok === false);
const gateFailures = gateResults.filter((item) => item.ok === false);
const status = job.status || {};
const metadata = job.metadata || {};
console.log(JSON.stringify({
ok: logFailures.length === 0,
degradedReason: logFailures.length === 0 ? null : "log-read-failed",
ok: logFailures.length === 0 && gateFailures.length === 0 && !(status.failed && status.failed > 0),
degradedReason: logFailures.length > 0 ? "log-read-failed" : gateFailures.length > 0 ? "gate-failed" : status.failed && status.failed > 0 ? "job-failed" : null,
errors: logFailures.map((item) => ({ pod: item.pod, container: item.container, degradedReason: item.degradedReason, message: item.message })),
query: { namespace, jobName, stage: stageName, sourceCommit: sourceCommit || null },
job: {
@@ -94,6 +98,7 @@ async function main() {
failedState: Boolean(status.failed && status.failed > 0),
},
pods: podSummaries,
gateResults,
logs,
nodeCicdTiming: lastTiming(logs),
statusAuthority: useServiceAccount ? "kubernetes-api-serviceaccount" : "target-node-kubectl-raw",
@@ -290,6 +295,145 @@ function lastTiming(logs) {
return null;
}
function parseLastJsonObject(text) {
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
for (let index = lines.length - 1; index >= 0; index -= 1) {
const line = lines[index];
if (!line.startsWith("{") || !line.endsWith("}")) continue;
try {
const parsed = JSON.parse(line);
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed;
} catch {
// Keep scanning bounded log lines.
}
}
return null;
}
function compactGateResult(value) {
if (!value || typeof value !== "object") return null;
const evidence = objectOrEmpty(value.evidence);
const closeout = objectOrEmpty(evidence.closeout);
const argo = objectOrEmpty(closeout.argo);
const runtime = objectOrEmpty(closeout.runtime);
const pipelineRun = objectOrEmpty(closeout.pipelineRun);
return {
ok: value.ok === true,
gate: stringOrNull(value.gate),
follower: stringOrNull(value.follower),
sourceCommit: stringOrNull(evidence.sourceCommit) || shortSha(stringOrNull(closeout.source?.sourceCommit)),
errors: arrayItems(value.errors).slice(0, 4).map((item) => shortText(item)),
polls: integerOrNull(evidence.polls),
elapsedMs: integerOrNull(evidence.elapsedMs),
writesState: booleanOrNull(evidence.writesState),
pipelineRun: {
name: stringOrNull(pipelineRun.name),
succeeded: booleanOrNull(pipelineRun.succeeded),
reason: stringOrNull(pipelineRun.reason),
durationSeconds: integerOrNull(pipelineRun.durationSeconds),
},
argo: {
syncStatus: stringOrNull(argo.syncStatus),
healthStatus: stringOrNull(argo.healthStatus),
operationPhase: stringOrNull(argo.operationPhase),
operationMessage: shortText(argo.operationMessage),
nonReadyResources: compactNamedResources(argo.nonReadyResources, 4),
syncResultResources: compactNamedResources(argo.syncResultResources, 3),
problemWorkloads: compactProblemWorkloads(argo.problemWorkloads, 3),
},
runtime: {
namespace: stringOrNull(runtime.namespace),
ready: booleanOrNull(runtime.ready),
aligned: booleanOrNull(runtime.aligned),
workloads: compactRuntimeWorkloads(runtime.workloads, 8),
},
statusAuthority: stringOrNull(value.statusAuthority),
parsedDownstreamCliOutput: value.parsedDownstreamCliOutput === true,
};
}
function compactLogTail(text, parsedGateResult) {
if (!text) return "";
if (parsedGateResult !== null) {
const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
const nonJson = lines.filter((line) => !(line.startsWith("{") && line.endsWith("}")));
return shortText(nonJson.slice(-8).join("\n"));
}
return shortText(text);
}
function compactNamedResources(value, limit) {
return arrayItems(value).slice(0, limit).map((item) => ({
kind: stringOrNull(item.kind),
namespace: stringOrNull(item.namespace),
name: stringOrNull(item.name),
status: stringOrNull(item.status),
healthStatus: stringOrNull(item.healthStatus),
hookPhase: stringOrNull(item.hookPhase),
syncPhase: stringOrNull(item.syncPhase),
message: shortText(item.message || item.healthMessage),
}));
}
function compactRuntimeWorkloads(value, limit) {
return arrayItems(value).slice(0, limit).map((item) => ({
kind: stringOrNull(item.kind),
name: stringOrNull(item.name),
ready: booleanOrNull(item.ready),
aligned: booleanOrNull(item.aligned),
desired: integerOrNull(item.desired),
readyReplicas: integerOrNull(item.readyReplicas),
updatedReplicas: integerOrNull(item.updatedReplicas),
sourceCommit: stringOrNull(item.sourceCommit),
}));
}
function compactProblemWorkloads(value, limit) {
return arrayItems(value).filter(isProblemWorkload).slice(0, limit).map((item) => ({
kind: stringOrNull(item.kind),
namespace: stringOrNull(item.namespace),
name: stringOrNull(item.name),
desired: integerOrNull(item.desired),
readyReplicas: integerOrNull(item.readyReplicas),
updatedReplicas: integerOrNull(item.updatedReplicas),
unavailableReplicas: integerOrNull(item.unavailableReplicas),
conditions: arrayItems(item.conditions).slice(0, 2).map((condition) => ({
type: stringOrNull(condition.type),
status: stringOrNull(condition.status),
reason: stringOrNull(condition.reason),
message: shortText(condition.message),
})),
pods: arrayItems(item.pods).filter(isProblemPod).slice(0, 2).map((pod) => ({
name: stringOrNull(pod.name),
phase: stringOrNull(pod.phase),
ready: booleanOrNull(pod.ready),
containers: arrayItems(pod.containers).filter(isProblemContainer).slice(0, 2).map((container) => ({
name: stringOrNull(container.name),
ready: booleanOrNull(container.ready),
restartCount: integerOrNull(container.restartCount),
waitingReason: stringOrNull(container.waitingReason),
terminatedReason: stringOrNull(container.terminatedReason),
exitCode: integerOrNull(container.exitCode),
})),
})),
}));
}
function isProblemWorkload(item) {
const desired = integerOrNull(item?.desired) ?? 1;
const readyReplicas = integerOrNull(item?.readyReplicas) ?? 0;
const unavailableReplicas = integerOrNull(item?.unavailableReplicas) ?? 0;
return readyReplicas < desired || unavailableReplicas > 0 || arrayItems(item?.pods).some(isProblemPod);
}
function isProblemPod(pod) {
return pod?.ready === false || arrayItems(pod?.containers).some(isProblemContainer);
}
function isProblemContainer(container) {
return container?.ready === false || Boolean(container?.waitingReason) || Boolean(container?.terminatedReason) || integerOrNull(container?.exitCode) !== null;
}
function durationSeconds(start, end) {
const s = timestampMs(start);
const e = timestampMs(end);
@@ -317,7 +461,8 @@ function shortSha(value) {
function shortText(value) {
if (value === null || value === undefined) return null;
const text = String(value).replace(/\s+/gu, " ").trim();
return text.length <= maxMessageBytes ? text : `${text.slice(0, Math.max(0, maxMessageBytes - 3))}...`;
const limit = Math.min(maxMessageBytes, 300);
return text.length <= limit ? text : `${text.slice(0, Math.max(0, limit - 3))}...`;
}
function tailBytes(value, maxBytes) {
@@ -330,6 +475,18 @@ function arrayItems(value) {
return Array.isArray(value) ? value : [];
}
function objectOrEmpty(value) {
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
}
function stringOrNull(value) {
return typeof value === "string" && value.length > 0 ? value : null;
}
function booleanOrNull(value) {
return typeof value === "boolean" ? value : null;
}
function integerOrNull(value) {
return Number.isInteger(value) ? value : null;
}