feat: add bounded cicd long polling
This commit is contained in:
Codex
@@ -14,7 +14,9 @@ import {
|
||||
sha256Fingerprint,
|
||||
} from "./platform-infra-ops-library";
|
||||
import { materializeYamlComposition } from "./yaml-composition";
|
||||
import { kubernetesWatchOneEventShellFunction } from "./kubernetes-watch";
|
||||
import { pacNodeReadOnlyNext, pacReadOnlyNext, resolveCicdDeliveryAuthority } from "./cicd-delivery-authority";
|
||||
import { DEFAULT_READ_ONLY_LONG_POLL_TIMEOUT_MS, parseStrictDuration, readOnlyLongPollTransportTimeoutMs, runReadOnlyLongPoll } from "./read-only-long-poll";
|
||||
import {
|
||||
canonicalizePacPipelineSpec,
|
||||
pacSourceArtifactSafeError,
|
||||
@@ -250,6 +252,7 @@ export interface PacHistoryConsumerSelection {
|
||||
interface CloseoutOptions extends CommonOptions {
|
||||
sourceCommit: string | null;
|
||||
wait: boolean;
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
interface ApplyOptions extends CommonOptions {
|
||||
@@ -291,7 +294,20 @@ export async function runPlatformInfraPipelinesAsCodeCommand(config: UniDeskConf
|
||||
return options.full || options.raw ? result : options.json ? compactStatusJson(result) : renderStatus(result);
|
||||
}
|
||||
if (action === "closeout") {
|
||||
const options = parseCloseoutOptions(args.slice(1));
|
||||
let options: CloseoutOptions;
|
||||
try {
|
||||
options = parseCloseoutOptions(args.slice(1));
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
action: "platform-infra-pipelines-as-code-closeout-validation",
|
||||
mutation: false,
|
||||
code: "validation-failed",
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
usage: "bun scripts/cli.ts platform-infra pipelines-as-code closeout --target <NODE> --consumer <consumer> --wait [--timeout 120s]",
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
const result = await closeout(config, options);
|
||||
return options.full || options.raw ? result : options.json ? compactCloseoutJson(result) : renderCloseout(result);
|
||||
}
|
||||
@@ -421,7 +437,7 @@ function help(scope: string | null): Record<string, unknown> {
|
||||
scope: "explicit-readonly-or-connectivity-diagnostics",
|
||||
configTruth: configLabel,
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code closeout --target <NODE> --consumer <consumer> --source-commit <sha> --wait",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code closeout --target <NODE> --consumer <consumer> --source-commit <sha> --wait [--timeout 120s]",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code status --target <NODE> --consumer <consumer> --full",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code history --target <NODE> --consumer <consumer> --limit 10 --full",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code debug-step --target <NODE> --consumer <consumer> --json",
|
||||
@@ -1398,22 +1414,25 @@ async function closeout(config: UniDeskConfig, options: CloseoutOptions): Promis
|
||||
const target = resolveTarget(pac, options.targetId);
|
||||
const consumer = resolveConsumer(pac, options.consumerId);
|
||||
const startedAt = Date.now();
|
||||
const waitMs = Math.max(1, pac.release.waitTimeoutSeconds) * 1000;
|
||||
let ciAttempts = 0;
|
||||
let runtimeAttempts = 0;
|
||||
let current = await observeCloseoutStatus(config, options, target, consumer, "ci", 1, startedAt);
|
||||
ciAttempts += 1;
|
||||
while (options.wait && !closeoutCiReady(current, options.sourceCommit) && Date.now() - startedAt < waitMs) {
|
||||
await sleep(3000);
|
||||
current = await observeCloseoutStatus(config, options, target, consumer, "ci", ciAttempts + 1, startedAt);
|
||||
ciAttempts += 1;
|
||||
}
|
||||
let current: Record<string, unknown> = {};
|
||||
let phase: "ci" | "runtime" = "ci";
|
||||
const waitResult = await runReadOnlyLongPoll({
|
||||
timeoutMs: options.wait ? options.timeoutMs : 1,
|
||||
observe: async () => {
|
||||
current = await observeCloseoutStatus(config, options, target, consumer, phase, 1, startedAt);
|
||||
phase = closeoutCiReady(current, options.sourceCommit) ? "runtime" : "ci";
|
||||
return current;
|
||||
},
|
||||
completed: (value) => !options.wait || closeoutWaitComplete(value, options.sourceCommit),
|
||||
waitForChange: async (remainingMs) => await waitForCloseoutChange(config, target, consumer, phase, remainingMs),
|
||||
cursor: closeoutCursor,
|
||||
});
|
||||
current = waitResult.value;
|
||||
let summary = record(current.summary);
|
||||
let latest = record(summary.latestPipelineRun);
|
||||
let diagnostics = record(summary.diagnostics);
|
||||
let observedSourceCommit = observedCloseoutSourceCommit(latest, diagnostics);
|
||||
let sourceMatched = options.sourceCommit === null || observedSourceCommit === options.sourceCommit;
|
||||
const ciReady = closeoutCiReady(current, options.sourceCommit);
|
||||
let gitOpsMirrorFlush: Record<string, unknown> = {
|
||||
ok: true,
|
||||
enabled: false,
|
||||
@@ -1426,16 +1445,6 @@ async function closeout(config: UniDeskConfig, options: CloseoutOptions): Promis
|
||||
configSource: configLabel,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
const runtimeStartedAt = Date.now();
|
||||
if (ciReady) {
|
||||
current = await observeCloseoutStatus(config, options, target, consumer, "runtime", 1, startedAt);
|
||||
runtimeAttempts += 1;
|
||||
while (options.wait && !closeoutReady(current, options.sourceCommit) && Date.now() - runtimeStartedAt < waitMs) {
|
||||
await sleep(3000);
|
||||
current = await observeCloseoutStatus(config, options, target, consumer, "runtime", runtimeAttempts + 1, startedAt);
|
||||
runtimeAttempts += 1;
|
||||
}
|
||||
}
|
||||
summary = record(current.summary);
|
||||
latest = record(summary.latestPipelineRun);
|
||||
diagnostics = record(summary.diagnostics);
|
||||
@@ -1452,8 +1461,8 @@ async function closeout(config: UniDeskConfig, options: CloseoutOptions): Promis
|
||||
ready,
|
||||
sourceMatched,
|
||||
diagnosticCode: diagnostics.code,
|
||||
ciAttempts,
|
||||
runtimeAttempts,
|
||||
timedOut: waitResult.timedOut,
|
||||
observations: waitResult.observations,
|
||||
});
|
||||
return {
|
||||
ok: ready,
|
||||
@@ -1465,26 +1474,66 @@ async function closeout(config: UniDeskConfig, options: CloseoutOptions): Promis
|
||||
observedSourceCommit,
|
||||
sourceMatched,
|
||||
ready,
|
||||
timedOut: waitResult.timedOut,
|
||||
elapsedMs: waitResult.elapsedMs,
|
||||
cursor: waitResult.cursor,
|
||||
wait: {
|
||||
requested: options.wait,
|
||||
attempts: ciAttempts + runtimeAttempts,
|
||||
ciAttempts,
|
||||
runtimeAttempts,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
budgetSeconds: pac.release.waitTimeoutSeconds,
|
||||
budgetScope: "per-phase",
|
||||
source: `${configLabel}.release.waitTimeoutSeconds`,
|
||||
completed: waitResult.completed,
|
||||
timedOut: waitResult.timedOut,
|
||||
cancelled: waitResult.cancelled,
|
||||
observations: waitResult.observations,
|
||||
elapsedMs: waitResult.elapsedMs,
|
||||
budgetSeconds: options.timeoutMs / 1000,
|
||||
budgetScope: "whole-observation",
|
||||
source: options.wait ? "cli --timeout or default 120s" : "single observation",
|
||||
identity: { target: target.id, consumer: consumer.id, pipelineRun: latest.name ?? null, sourceCommit: options.sourceCommit, valuesPrinted: false },
|
||||
cursor: waitResult.cursor,
|
||||
valuesPrinted: false,
|
||||
},
|
||||
summary,
|
||||
status: current,
|
||||
gitOpsMirrorFlush,
|
||||
blocker: ready ? null : closeoutBlocker(current, options.sourceCommit, observedSourceCommit, sourceMatched, gitOpsMirrorFlush),
|
||||
next: nextCommands(target.id, consumer.id, pac.defaults.consumerId),
|
||||
next: {
|
||||
...nextCommands(target.id, consumer.id, pac.defaults.consumerId),
|
||||
wait: `bun scripts/cli.ts platform-infra pipelines-as-code closeout --target ${target.id} --consumer ${consumer.id}${options.sourceCommit === null ? "" : ` --source-commit ${options.sourceCommit}`} --wait --timeout 120s`,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForCloseoutChange(config: UniDeskConfig, target: PacTarget, consumer: PacConsumer, phase: "ci" | "runtime", remainingMs: number): Promise<void> {
|
||||
const timeoutSeconds = Math.max(1, Math.ceil(remainingMs / 1000));
|
||||
const args = phase === "ci"
|
||||
? ["kubectl", "-n", consumer.namespace, "get", "pipelinerun", "-l", `tekton.dev/pipeline=${consumer.pipeline}`, "--watch-only", `--request-timeout=${timeoutSeconds}s`, "-o", "name"]
|
||||
: ["kubectl", "-n", consumer.argoNamespace, "get", "application", consumer.argoApplication, "--watch-only", `--request-timeout=${timeoutSeconds}s`, "-o", "name"];
|
||||
const script = [
|
||||
"set -euo pipefail",
|
||||
kubernetesWatchOneEventShellFunction(),
|
||||
`unidesk_watch_one_event ${args.map(shQuote).join(" ")}`,
|
||||
].join("\n");
|
||||
const result = await capture(config, target.route, ["bash"], script, { runtimeTimeoutMs: readOnlyLongPollTransportTimeoutMs(remainingMs) });
|
||||
if (result.exitCode !== 0) throw new Error(`CI/CD ${phase} watch disconnected: exit=${result.exitCode} stderr=${stringValue(compactCapture(result, { full: true }).stderrTail)}`);
|
||||
}
|
||||
|
||||
function closeoutWaitComplete(value: Record<string, unknown>, sourceCommit: string | null): boolean {
|
||||
if (closeoutReady(value, sourceCommit)) return true;
|
||||
const latest = record(record(value.summary).latestPipelineRun);
|
||||
const terminal = stringValue(latest.status, stringValue(latest.reason, "")).toLowerCase();
|
||||
return ["failed", "failure", "false", "cancelled", "canceled"].some((status) => terminal.includes(status));
|
||||
}
|
||||
|
||||
function closeoutCursor(value: Record<string, unknown>): string {
|
||||
const summary = record(value.summary);
|
||||
const latest = record(summary.latestPipelineRun);
|
||||
const argo = record(summary.argo);
|
||||
const runtime = record(summary.runtime);
|
||||
return [latest.name, latest.status, latest.reason, argo.revision, argo.sync, argo.health, runtime.digest, runtime.readyReplicas]
|
||||
.map((item) => stringValue(item, "-"))
|
||||
.join(":");
|
||||
}
|
||||
|
||||
async function observeCloseoutStatus(config: UniDeskConfig, options: CloseoutOptions, target: PacTarget, consumer: PacConsumer, phase: "ci" | "runtime", attempt: number, startedAt: number): Promise<Record<string, unknown>> {
|
||||
printCloseoutProgress(target, consumer, `${phase}-status`, "started", startedAt, { attempt });
|
||||
const current = await status(config, options);
|
||||
@@ -2324,6 +2373,9 @@ function compactCloseoutJson(result: Record<string, unknown>): Record<string, un
|
||||
observedSourceCommit: result.observedSourceCommit,
|
||||
sourceMatched: result.sourceMatched === true,
|
||||
ready: result.ready === true,
|
||||
timedOut: result.timedOut === true,
|
||||
elapsedMs: result.elapsedMs,
|
||||
cursor: result.cursor,
|
||||
wait: result.wait,
|
||||
summary: compactStatusSummary(record(result.summary)),
|
||||
gitOpsMirrorFlush: {
|
||||
@@ -2655,15 +2707,17 @@ function renderCloseout(result: Record<string, unknown>): RenderedCliResult {
|
||||
...table(["TARGET", "CONSUMER", "READY", "SOURCE_MATCHED", "MUTATION"], [[stringValue(target.id), stringValue(consumer.id), boolText(result.ready), boolText(result.sourceMatched), boolText(result.mutation)]]),
|
||||
"",
|
||||
"WAIT",
|
||||
...table(["REQUESTED", "CI_ATTEMPTS", "RUNTIME_ATTEMPTS", "ELAPSED_MS", "BUDGET_S", "SCOPE", "SOURCE"], [[
|
||||
...table(["REQUESTED", "COMPLETED", "TIMED_OUT", "CANCELLED", "OBSERVATIONS", "ELAPSED_MS", "BUDGET_S"], [[
|
||||
stringValue(wait.requested),
|
||||
stringValue(wait.ciAttempts),
|
||||
stringValue(wait.runtimeAttempts),
|
||||
boolText(wait.completed),
|
||||
boolText(wait.timedOut),
|
||||
boolText(wait.cancelled),
|
||||
stringValue(wait.observations),
|
||||
stringValue(wait.elapsedMs),
|
||||
stringValue(wait.budgetSeconds),
|
||||
stringValue(wait.budgetScope),
|
||||
stringValue(wait.source),
|
||||
]]),
|
||||
` identity: ${compactLine(JSON.stringify(wait.identity ?? {}))}`,
|
||||
` cursor: ${stringValue(wait.cursor)}`,
|
||||
"",
|
||||
"SOURCE / PIPELINERUN",
|
||||
...table(["EXPECTED", "OBSERVED", "PIPELINERUN", "STATUS", "DURATION_S"], [[short(stringValue(result.sourceCommit), 16), short(stringValue(result.observedSourceCommit), 16), stringValue(latest.name), stringValue(latest.reason ?? latest.status), stringValue(latest.durationSeconds)]]),
|
||||
@@ -2701,6 +2755,7 @@ function renderCloseout(result: Record<string, unknown>): RenderedCliResult {
|
||||
"NEXT",
|
||||
` status: ${stringValue(record(result.next).status)}`,
|
||||
` history: ${stringValue(record(result.next).history)} --limit 10`,
|
||||
` wait: ${stringValue(record(result.next).wait)}`,
|
||||
latest.name === undefined ? " pipeline-run: -" : ` pipeline-run: ${stringValue(record(result.next).history)} --id ${stringValue(latest.name)}`,
|
||||
];
|
||||
return rendered(result, "platform-infra pipelines-as-code closeout", lines);
|
||||
@@ -2904,6 +2959,7 @@ function parseCloseoutOptions(args: string[]): CloseoutOptions {
|
||||
const commonArgs: string[] = [];
|
||||
let sourceCommit: string | null = null;
|
||||
let wait = false;
|
||||
let timeoutMs = DEFAULT_READ_ONLY_LONG_POLL_TIMEOUT_MS;
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--source-commit" || arg === "--commit") {
|
||||
@@ -2914,6 +2970,11 @@ function parseCloseoutOptions(args: string[]): CloseoutOptions {
|
||||
index += 1;
|
||||
} else if (arg === "--wait") {
|
||||
wait = true;
|
||||
} else if (arg === "--timeout") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error("--timeout requires a value");
|
||||
timeoutMs = parseStrictDuration(value, "--timeout");
|
||||
index += 1;
|
||||
} else {
|
||||
commonArgs.push(arg);
|
||||
if (arg === "--target" || arg === "--node" || arg === "--consumer") {
|
||||
@@ -2922,7 +2983,7 @@ function parseCloseoutOptions(args: string[]): CloseoutOptions {
|
||||
}
|
||||
}
|
||||
}
|
||||
return { ...parseCommonOptions(commonArgs), sourceCommit, wait };
|
||||
return { ...parseCommonOptions(commonArgs), sourceCommit, wait, timeoutMs };
|
||||
}
|
||||
|
||||
function parseHistoryOptions(args: string[]): HistoryOptions {
|
||||
@@ -3065,9 +3126,6 @@ function closeoutBlocker(value: Record<string, unknown>, expected: string | null
|
||||
};
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
|
||||
function pipelineRunGate(latest: Record<string, unknown>): Record<string, unknown> {
|
||||
|
||||
Reference in New Issue
Block a user