fix: reread follower closeout after accelerators

2026-07-03 19:41:39 +00:00
parent e9cc4f8ed3
commit ba09dfa8b2
2 changed files with 35 additions and 11 deletions
@@ -58,6 +58,8 @@ The automatic controller loop is non-blocking, so closeout acceleration cannot l

 The same rule applies to git-mirror post-flush. If native status shows runtime/Argo are aligned but GitOps mirror is still pending flush, the automatic controller loop must run the bounded target-side git-mirror flush instead of leaving a follower in `ClosingOut` until a manual wait/closeout path is used.

+After an automatic closeout accelerator runs, the same reconcile must do one bounded native status re-read and write the resulting state when it is already aligned. Do not defer the final `Noop` write to the next controller loop; loop interval plus another status-read can add enough idle time to exceed the 120s end-to-end budget even when PipelineRun, Argo and runtime are already ready. The re-read timeout must come from YAML follower budgets.
+
 Stage timing rows must not label optional gates as `not-ready` when they are not part of that follower's closeout contract. For sentinel-like followers without a GitOps branch flush gate, git-mirror source snapshot readiness should render as source-ready/ready, while missing GitOps `githubInSync` remains `-`/not-applicable instead of a failure-looking state.

 ## Source Authority
@@ -859,17 +859,37 @@ async function decideAndMaybeTrigger(
  }

  if (options.dryRun && phase === "PendingTrigger") decision = `${decision}; dry-run did not trigger`;
+  let stateLive: AdapterSummary = live;
+  let automaticCloseoutAccelerated = false;
  if (shouldRefreshAutomaticCloseout(follower, observedSha, live, phase, options)) {
-    const refresh = runNativeArgoRefresh(follower.nativeStatus.argo as NonNullable<NativeStatusSpec["argo"]>);
+    const refresh = runNativeArgoRefresh(follower.nativeStatus.argo as NonNullable<NativeStatusSpec["argo"]>, follower.budgets.controlPlaneRefreshSeconds);
    if (refresh.exitCode !== 0) warnings.push(`argo refresh failed: ${redactText(tailText(refresh.stderr || refresh.stdout, 300))}`);
+    else automaticCloseoutAccelerated = true;
  }
  if (shouldFlushAutomaticCloseout(follower, observedSha, live, phase, options)) {
    const gitMirror = asOptionalRecord(asOptionalRecord(live.payload)?.gitMirror);
    const flushKey = stringOrNull(gitMirror?.localGitops) ?? observedSha;
    const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", follower.budgets.sourceSyncSeconds, flushKey);
    if (flush !== null && !flush.result.ok) warnings.push(`git-mirror flush failed: ${redactText(tailText(flush.result.conditionMessage ?? flush.result.logsTail ?? "unknown", 300))}`);
+    else if (flush !== null) automaticCloseoutAccelerated = true;
  }
-  const statePipelineRun = stringOrNull(triggerCommand?.pipelineRun) ?? live.pipelineRun;
+  if (automaticCloseoutAccelerated && observedSha !== null) {
+    const reread = await readAdapterStatus(registry, follower, { ...options, timeoutSeconds: follower.budgets.statusSeconds });
+    if (!reread.ok) {
+      warnings.push(`post-closeout status re-read failed: ${redactText(tailText(reread.message, 300))}`);
+    } else if (reread.observedSha === observedSha) {
+      stateLive = reread;
+      targetSha = reread.targetSha ?? targetSha;
+      inFlightJob = reread.inFlightJob;
+      if (reread.aligned === true) {
+        phase = "Noop";
+        decision = "target already matches observed source sha";
+        inFlightJob = null;
+        lastSucceededSha = observedSha;
+      }
+    }
+  }
+  const statePipelineRun = stringOrNull(triggerCommand?.pipelineRun) ?? stateLive.pipelineRun;

  return {
    id: follower.id,
@@ -881,7 +901,7 @@ async function decideAndMaybeTrigger(
      branch: follower.source.branch,
      branchRef: follower.source.branchRef,
      snapshotPrefix: follower.source.snapshotPrefix,
-      observedSha,
+      observedSha: stateLive.observedSha ?? observedSha,
    },
    target: {
      node: follower.target.node,
@@ -903,14 +923,14 @@ async function decideAndMaybeTrigger(
    decision,
    dryRun: options.dryRun,
    updatedAt: new Date().toISOString(),
-    timings: buildFollowerTimings(follower, live, triggerCommand, asOptionalRecord(previous.timings), phase),
+    timings: buildFollowerTimings(follower, stateLive, triggerCommand, asOptionalRecord(previous.timings), phase),
    warnings,
    next: followerNextCommands(follower),
    command: triggerCommand ?? {
-      status: live.command,
-      exitCode: live.exitCode,
-      timedOut: live.timedOut,
-      payload: live.payload,
+      status: stateLive.command,
+      exitCode: stateLive.exitCode,
+      timedOut: stateLive.timedOut,
+      payload: stateLive.payload,
    },
  };
 }
@@ -1362,7 +1382,9 @@ async function waitNativeSentinelCloseout(
  options: ParsedOptions,
  timeoutSeconds: number,
 ): Promise<NativeCloseoutWaitResult> {
-  const refreshResult = follower.nativeStatus.argo === null ? null : runNativeArgoRefresh(follower.nativeStatus.argo);
+  const refreshResult = follower.nativeStatus.argo === null
+    ? null
+    : runNativeArgoRefresh(follower.nativeStatus.argo, Math.min(timeoutSeconds, follower.budgets.controlPlaneRefreshSeconds));
  const startedAt = Date.now();
  const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000;
  let polls = 0;
@@ -1501,7 +1523,7 @@ function nativeCloseoutSummary(live: AdapterSummary): Record<string, unknown> {
  };
 }

-function runNativeArgoRefresh(argo: NonNullable<NativeStatusSpec["argo"]>): CommandResult {
+function runNativeArgoRefresh(argo: NonNullable<NativeStatusSpec["argo"]>, timeoutSeconds: number): CommandResult {
  const patchBase64 = Buffer.from(JSON.stringify({
    metadata: {
      annotations: {
@@ -1540,7 +1562,7 @@ function runNativeArgoRefresh(argo: NonNullable<NativeStatusSpec["argo"]>): Comm
    "req.end();",
    "NODE_ARGO_REFRESH",
  ].join("\n");
-  return runCommand(["sh", "-lc", script], repoRoot, { timeoutMs: 10_000 });
+  return runCommand(["sh", "-lc", script], repoRoot, { timeoutMs: Math.max(1, timeoutSeconds) * 1000 });
 }

 async function readAdapterStatus(registry: BranchFollowerRegistry, follower: FollowerSpec, options: ParsedOptions): Promise<AdapterSummary> {