fix: bound node runtime cicd wait closeout

2026-07-02 04:38:46 +00:00
parent 1b24e994f8
commit ed32eb84b2
12 changed files with 880 additions and 409 deletions
@@ -38,6 +38,12 @@ import { webObserveTable } from "./web-observe-render";
 import { nodeRuntimeGitMirrorTarget, printNodeRuntimeTriggerProgress, sleepSync } from "./web-probe";
 import { webObserveShort, webObserveText } from "./web-probe-observe";

+type NodeScopedDelegatedOptions = ReturnType<typeof parseNodeScopedDelegatedOptions>;
+export type NodeRuntimeGitMirrorRunOptions = {
+  maxAttempts?: number;
+  deadlineMs?: number;
+};
+
 export function nodeRuntimeExternalPostgresSecretRows(secrets: Record<string, unknown>): unknown[][] {
  if (secrets.required === false) return [["not-required", "-", "-", "-", "false"]];
  if (secrets.required === true && secrets.ready !== true && secrets.degradedReason !== undefined) {
@@ -240,12 +246,12 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeSc
  };
 }

-export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
+export function nodeRuntimeGitMirrorRun(scoped: NodeScopedDelegatedOptions, options: NodeRuntimeGitMirrorRunOptions = {}): Record<string, unknown> {
  if (scoped.action !== "sync" && scoped.action !== "flush") return nodeRuntimeUnsupportedAction(scoped);
  if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`);
  const spec = scoped.spec;
  const mirror = nodeRuntimeGitMirrorTarget(spec);
-  const retryMaxAttempts = !scoped.dryRun && (scoped.action === "sync" || scoped.action === "flush") ? 5 : 1;
+  const retryMaxAttempts = options.maxAttempts ?? (!scoped.dryRun && (scoped.action === "sync" || scoped.action === "flush") ? 5 : 1);
  const attempts: Record<string, unknown>[] = [];
  let finalAttempt: {
    attempt: number;
@@ -259,13 +265,15 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
  } | null = null;

  for (let attempt = 1; attempt <= retryMaxAttempts; attempt += 1) {
+    const attemptScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs, 5);
+    if (attemptScoped === null) break;
    const retryLabel = `${attempt}/${retryMaxAttempts}`;
    const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action);
    const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName, {
      discardStaleGitops: scoped.discardStaleGitops === true,
    });
    const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
-    const waitTimeoutSeconds = Math.max(5, Math.min(45, Math.max(5, scoped.timeoutSeconds - 10)));
+    const waitTimeoutSeconds = Math.max(1, Math.min(45, Math.max(1, attemptScoped.timeoutSeconds - 10)));
    const script = [
      "set -eu",
      `namespace=${shellQuote(mirror.namespace)}`,
@@ -291,7 +299,7 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
          "kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true",
        ].join("\n"),
    ].join("\n");
-    const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
+    const result = runNodeK3sScript(spec, script, attemptScoped.timeoutSeconds);
    const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
    const actionSucceeded = isCommandSuccess(result);
    const retryableFailure = !actionSucceeded
@@ -317,6 +325,7 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
    finalAttempt = { attempt, retryLabel, jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded };
    if (actionSucceeded || scoped.dryRun || !retryable || exhausted) break;
    const backoffMs = nodeRuntimeGitMirrorRetryDelayMs(attempt);
+    if (options.deadlineMs !== undefined && Date.now() + backoffMs >= options.deadlineMs) break;
    printNodeRuntimeTriggerProgress(spec, {
      stage: `git-mirror-${scoped.action}-retry`,
      status: "waiting",
@@ -328,11 +337,14 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
    sleepSync(backoffMs);
  }

-  if (finalAttempt === null) throw new Error("git-mirror run produced no attempts");
+  if (finalAttempt === null) {
+    return nodeRuntimeGitMirrorBudgetExhausted(scoped);
+  }
  const { jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded } = finalAttempt;
-  const status = scoped.dryRun || !actionSucceeded
+  const statusScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs, 5);
+  const status = scoped.dryRun || !actionSucceeded || statusScoped === null
    ? undefined
-    : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
+    : nodeRuntimeGitMirrorStatus({ ...statusScoped, action: "status", dryRun: true, confirm: false });
  const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts;
  const stopped = !actionSucceeded && (retryExhausted || retryableFailure?.stopped === true || retryableFailure === null);
  return {
@@ -348,6 +360,7 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
    manifest: scoped.dryRun ? manifest : undefined,
    result: compactRuntimeCommand(result),
    status,
+    statusSkippedReason: statusScoped === null && !scoped.dryRun && actionSucceeded ? "node-runtime-cicd-budget-exhausted" : undefined,
    retry: retryMaxAttempts > 1 ? {
      policy: "exponential-backoff",
      maxAttempts: retryMaxAttempts,
@@ -370,6 +383,27 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
  };
 }

+function nodeRuntimeScopedWithinDeadline(scoped: NodeScopedDelegatedOptions, deadlineMs?: number, minRemainingSeconds = 5): NodeScopedDelegatedOptions | null {
+  if (deadlineMs === undefined) return scoped;
+  const remainingSeconds = Math.floor((deadlineMs - Date.now()) / 1000);
+  if (remainingSeconds < minRemainingSeconds) return null;
+  return { ...scoped, timeoutSeconds: Math.max(1, Math.min(scoped.timeoutSeconds, remainingSeconds)) };
+}
+
+function nodeRuntimeGitMirrorBudgetExhausted(scoped: NodeScopedDelegatedOptions): Record<string, unknown> {
+  return {
+    ok: false,
+    command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
+    node: scoped.node,
+    lane: scoped.lane,
+    mode: `confirmed-${scoped.action}`,
+    mutation: false,
+    status: "pending",
+    degradedReason: "node-runtime-cicd-budget-exhausted",
+    next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
+  };
+}
+
 export function nodeRuntimeGitMirrorRetryableFailure(
  scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
  mirror: NodeRuntimeGitMirrorTargetSpec,
@@ -480,9 +514,20 @@ export function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mir
  return `http://${value}/${mirror.sourceRepository}.git`;
 }

-export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
+export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: NodeScopedDelegatedOptions, sourceCommit: string, pipelineRun: string, options: NodeRuntimeGitMirrorRunOptions = {}): Record<string, unknown> {
  const full = nodeScopedFullOutput(scoped);
-  const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
+  const beforeScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
+  if (beforeScoped === null) {
+    return {
+      ok: false,
+      mode: "budget-exhausted-before-status",
+      sourceCommit,
+      status: "pending",
+      degradedReason: "node-runtime-cicd-budget-exhausted",
+      next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
+    };
+  }
+  const before = nodeRuntimeGitMirrorStatus({ ...beforeScoped, action: "status", dryRun: true, confirm: false });
  const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
  const beforeGithubTransport = record(before.githubTransport);
  if (beforeGithubTransport.required === true && beforeGithubTransport.ready === false) {
@@ -501,7 +546,7 @@ export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeo
    };
  }
  if (before.ok === true && beforeSummary.localSource === sourceCommit && beforeSummary.githubSource === sourceCommit) {
-    const flush = nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, before);
+    const flush = nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, before, options);
    return {
      ok: flush.ok === true,
      mode: "already-current",
@@ -512,19 +557,32 @@ export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeo
      degradedReason: flush.ok === true ? undefined : "node-runtime-git-mirror-pre-flush-failed",
    };
  }
+  const syncScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
+  if (syncScoped === null) {
+    return {
+      ok: false,
+      mode: "budget-exhausted-before-sync",
+      sourceCommit,
+      beforeSummary,
+      before: full ? before : undefined,
+      status: "pending",
+      degradedReason: "node-runtime-cicd-budget-exhausted",
+      next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
+    };
+  }
  const sync = nodeRuntimeGitMirrorRun({
-    ...scoped,
+    ...syncScoped,
    domain: "git-mirror",
    action: "sync",
    confirm: true,
    dryRun: false,
    wait: true,
    discardStaleGitops: scoped.discardStaleGitops === true || scoped.rerun === true,
-  });
+  }, options);
  const after = record(sync.status);
  const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
  const sourceOk = sync.ok === true && afterSummary.localSource === sourceCommit && afterSummary.githubSource === sourceCommit;
-  const flush = sourceOk ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, after) : null;
+  const flush = sourceOk ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, after, options) : null;
  const ok = sourceOk && (flush === null || flush.ok === true);
  return {
    ok,
@@ -545,15 +603,28 @@ export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeo
 }

 export function nodeRuntimeEnsureGitMirrorFlushed(
-  scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
+  scoped: NodeScopedDelegatedOptions,
  phase: "pre" | "post" | "parallel",
  sourceCommit: string,
  pipelineRun: string | null,
  statusInput: Record<string, unknown> | null = null,
+  options: NodeRuntimeGitMirrorRunOptions = {},
 ): Record<string, unknown> {
  const stage = `git-mirror-${phase}-flush`;
  const full = nodeScopedFullOutput(scoped);
-  const before = statusInput ?? nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
+  const beforeScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
+  if (beforeScoped === null) {
+    return {
+      ok: false,
+      phase,
+      mode: "budget-exhausted-before-status",
+      executed: false,
+      status: "pending",
+      degradedReason: "node-runtime-cicd-budget-exhausted",
+      next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
+    };
+  }
+  const before = statusInput ?? nodeRuntimeGitMirrorStatus({ ...beforeScoped, action: "status", dryRun: true, confirm: false });
  const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
  if (before.ok !== true) {
    printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "failed", sourceCommit, pipelineRun, reason: "git-mirror-status-failed", ...beforeSummary });
@@ -583,7 +654,21 @@ export function nodeRuntimeEnsureGitMirrorFlushed(
    };
  }
  printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun, flushNeeded: true, ...beforeSummary });
-  const flush = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "flush", confirm: true, dryRun: false, wait: true });
+  const flushScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
+  if (flushScoped === null) {
+    return {
+      ok: false,
+      phase,
+      mode: "budget-exhausted-before-flush",
+      executed: false,
+      before: full ? before : undefined,
+      beforeSummary,
+      status: "pending",
+      degradedReason: "node-runtime-cicd-budget-exhausted",
+      next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
+    };
+  }
+  const flush = nodeRuntimeGitMirrorRun({ ...flushScoped, domain: "git-mirror", action: "flush", confirm: true, dryRun: false, wait: true }, options);
  const after = record(flush.status);
  const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
  const ok = flush.ok === true && Object.keys(after).length > 0 && !nodeRuntimeGitMirrorNeedsFlush(after);
@@ -613,14 +698,17 @@ export function nodeRuntimeEnsureGitMirrorFlushed(
 }

 export function nodeRuntimeOpportunisticGitMirrorSync(
-  scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
+  scoped: NodeScopedDelegatedOptions,
  sourceCommit: string,
  pipelineRun: string,
+  options: NodeRuntimeGitMirrorRunOptions = {},
 ): Record<string, unknown> | null {
  const stage = "git-mirror-parallel-sync";
+  const syncScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
+  if (syncScoped === null) return null;
  const full = nodeScopedFullOutput(scoped);
  printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun });
-  const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true });
+  const sync = nodeRuntimeGitMirrorRun({ ...syncScoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true }, options);
  const after = record(sync.status);
  const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
  const ok = sync.ok === true;
@@ -649,13 +737,16 @@ export function nodeRuntimeOpportunisticGitMirrorSync(
 }

 export function nodeRuntimeOpportunisticGitMirrorFlush(
-  scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
+  scoped: NodeScopedDelegatedOptions,
  sourceCommit: string,
  pipelineRun: string,
+  options: NodeRuntimeGitMirrorRunOptions = {},
 ): Record<string, unknown> | null {
-  const status = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
+  const statusScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
+  if (statusScoped === null) return null;
+  const status = nodeRuntimeGitMirrorStatus({ ...statusScoped, action: "status", dryRun: true, confirm: false });
  if (status.ok !== true || !nodeRuntimeGitMirrorNeedsFlush(status)) return null;
-  return nodeRuntimeEnsureGitMirrorFlushed(scoped, "parallel", sourceCommit, pipelineRun, status);
+  return nodeRuntimeEnsureGitMirrorFlushed(scoped, "parallel", sourceCommit, pipelineRun, status, options);
 }

 export function nodeRuntimeGitMirrorNeedsFlush(status: Record<string, unknown>): boolean {