Merge pull request #1504 from pikasTech/codex/agentrun-closeout-1501
fix(cicd): record automatic closeout timing
This commit is contained in:
@@ -952,28 +952,41 @@ async function decideAndMaybeTrigger(
|
||||
|
||||
if (options.dryRun && phase === "PendingTrigger") decision = `${decision}; dry-run did not trigger`;
|
||||
let stateLive: AdapterSummary = live;
|
||||
let automaticCloseoutStartedAt: number | null = null;
|
||||
let automaticCloseoutPolls = 0;
|
||||
let automaticCloseoutRefresh: Record<string, unknown> | null = null;
|
||||
let automaticCloseoutGitMirrorFlush: Record<string, unknown> | null = null;
|
||||
let automaticCloseoutAccelerated = false;
|
||||
if (shouldRefreshAutomaticCloseout(follower, observedSha, live, phase, options)) {
|
||||
automaticCloseoutStartedAt ??= Date.now();
|
||||
const refresh = runNativeArgoRefresh(follower.nativeStatus.argo as NonNullable<NativeStatusSpec["argo"]>, follower.budgets.controlPlaneRefreshSeconds);
|
||||
automaticCloseoutRefresh = commandCompact(refresh, options);
|
||||
if (refresh.exitCode !== 0) warnings.push(`argo refresh failed: ${redactText(tailText(refresh.stderr || refresh.stdout, 300))}`);
|
||||
else automaticCloseoutAccelerated = true;
|
||||
}
|
||||
if (shouldFlushAutomaticCloseout(follower, observedSha, live, phase, options)) {
|
||||
automaticCloseoutStartedAt ??= Date.now();
|
||||
const gitMirror = asOptionalRecord(asOptionalRecord(live.payload)?.gitMirror);
|
||||
const flushKey = stringOrNull(gitMirror?.localGitops) ?? observedSha;
|
||||
const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", follower.budgets.sourceSyncSeconds, flushKey);
|
||||
automaticCloseoutGitMirrorFlush = flush === null ? null : {
|
||||
jobName: flush.jobName,
|
||||
namespace: flush.namespace,
|
||||
result: compactNativeK8sJobResult(flush.result),
|
||||
};
|
||||
if (flush !== null && !flush.result.ok) warnings.push(`git-mirror flush failed: ${redactText(tailText(flush.result.conditionMessage ?? flush.result.logsTail ?? "unknown", 300))}`);
|
||||
else if (flush !== null) automaticCloseoutAccelerated = true;
|
||||
}
|
||||
if (automaticCloseoutAccelerated && observedSha !== null) {
|
||||
const reread = await readAdapterStatusAfterCloseoutAcceleration(registry, follower, observedSha, options);
|
||||
if (!reread.ok) {
|
||||
warnings.push(`post-closeout status re-read failed: ${redactText(tailText(reread.message, 300))}`);
|
||||
} else if (reread.observedSha === observedSha) {
|
||||
stateLive = reread;
|
||||
targetSha = reread.targetSha ?? targetSha;
|
||||
inFlightJob = reread.inFlightJob;
|
||||
if (reread.aligned === true) {
|
||||
automaticCloseoutPolls = reread.polls;
|
||||
if (!reread.latest.ok) {
|
||||
warnings.push(`post-closeout status re-read failed: ${redactText(tailText(reread.latest.message, 300))}`);
|
||||
} else if (reread.latest.observedSha === observedSha) {
|
||||
stateLive = reread.latest;
|
||||
targetSha = reread.latest.targetSha ?? targetSha;
|
||||
inFlightJob = reread.latest.inFlightJob;
|
||||
if (reread.latest.aligned === true) {
|
||||
phase = "Noop";
|
||||
decision = "target already matches observed source sha";
|
||||
inFlightJob = null;
|
||||
@@ -982,6 +995,21 @@ async function decideAndMaybeTrigger(
|
||||
}
|
||||
}
|
||||
const statePipelineRun = stringOrNull(triggerCommand?.pipelineRun) ?? stateLive.pipelineRun;
|
||||
if (automaticCloseoutStartedAt !== null && observedSha !== null && triggerCommand === undefined) {
|
||||
const completed = phase === "Noop" && targetSha === observedSha && lastSucceededSha === observedSha;
|
||||
triggerCommand = closeoutOnlyCommand(follower, statePipelineRun, observedSha, {
|
||||
ok: completed,
|
||||
completed,
|
||||
timedOut: false,
|
||||
polls: automaticCloseoutPolls,
|
||||
elapsedMs: Date.now() - automaticCloseoutStartedAt,
|
||||
refresh: automaticCloseoutRefresh,
|
||||
gitMirrorFlush: automaticCloseoutGitMirrorFlush,
|
||||
summary: nativeCloseoutSummary(stateLive),
|
||||
statusAuthority: "k8s-native",
|
||||
parsedDownstreamCliOutput: false,
|
||||
}, false);
|
||||
}
|
||||
|
||||
return {
|
||||
id: follower.id,
|
||||
@@ -1051,10 +1079,11 @@ async function readAdapterStatusAfterCloseoutAcceleration(
|
||||
follower: FollowerSpec,
|
||||
observedSha: string,
|
||||
options: ParsedOptions,
|
||||
): Promise<AdapterSummary> {
|
||||
): Promise<{ latest: AdapterSummary; polls: number; elapsedMs: number }> {
|
||||
const timeoutSeconds = follower.budgets.statusSeconds;
|
||||
const startedAt = Date.now();
|
||||
const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000;
|
||||
let polls = 1;
|
||||
let latest = await readAdapterStatus(registry, follower, { ...options, timeoutSeconds: Math.min(timeoutSeconds, remainingSeconds(startedAt, timeoutSeconds)) });
|
||||
while (
|
||||
latest.ok
|
||||
@@ -1067,9 +1096,10 @@ async function readAdapterStatusAfterCloseoutAcceleration(
|
||||
if (pollSeconds <= 0) break;
|
||||
runCommand(["sleep", String(pollSeconds)], repoRoot, { timeoutMs: (pollSeconds + 1) * 1000 });
|
||||
if (Date.now() >= deadline) break;
|
||||
polls += 1;
|
||||
latest = await readAdapterStatus(registry, follower, { ...options, timeoutSeconds: remainingSeconds(startedAt, timeoutSeconds) });
|
||||
}
|
||||
return latest;
|
||||
return { latest, polls, elapsedMs: Date.now() - startedAt };
|
||||
}
|
||||
|
||||
function shouldFlushAutomaticCloseout(
|
||||
@@ -1699,13 +1729,13 @@ async function waitNativeFollowerCloseout(
|
||||
return await waitNativeSentinelCloseout(registry, follower, observedSha, options, timeoutSeconds);
|
||||
}
|
||||
|
||||
function closeoutOnlyCommand(follower: FollowerSpec, pipelineRun: string | null, observedSha: string, closeout: NativeCloseoutWaitResult): Record<string, unknown> {
|
||||
function closeoutOnlyCommand(follower: FollowerSpec, pipelineRun: string | null, observedSha: string, closeout: NativeCloseoutWaitResult, wait = true): Record<string, unknown> {
|
||||
return {
|
||||
mode: "k8s-native-closeout",
|
||||
adapter: follower.adapter,
|
||||
pipelineRun,
|
||||
sourceCommit: observedSha,
|
||||
wait: true,
|
||||
wait,
|
||||
closeout,
|
||||
finishedAt: closeout.completed || closeout.timedOut ? new Date().toISOString() : null,
|
||||
elapsedMs: closeout.elapsedMs,
|
||||
|
||||
Reference in New Issue
Block a user