fix(cli): retry codex steer tunnel aborts

2026-05-23 06:28:13 +00:00
parent d0c762da06
commit 6c02cb1655
7 changed files with 196 additions and 23 deletions
@@ -33,6 +33,10 @@ const submitLockWaitMs = 60_000;
 const submitLockPollMs = 250;
 const submitLockStaleMs = 120_000;
 const submitThrottleMs = nonNegativeIntegerEnv("UNIDESK_CODEX_SUBMIT_THROTTLE_MS", 2000);
+const defaultSteerRetryAttempts = 2;
+const maxSteerRetryAttempts = 3;
+const defaultSteerRetryDelayMs = 750;
+const maxSteerRetryDelayMs = 5_000;

 interface CodexTaskOptions {
  detail: boolean;
@@ -138,6 +142,8 @@ interface SubmitRoutingRecommendation {
 interface CodexSteerOptions {
  prompt: string;
  dryRun: boolean;
+  retryAttempts: number;
+  retryDelayMs: number;
 }

 type CodexSteerFailureReason =
@@ -164,6 +170,19 @@ interface ClassifiedCodexSteerError {
  recommendedCrossChecks: string[];
 }

+interface CodexSteerAttemptSummary {
+  attempt: number;
+  ok: boolean;
+  durationMs: number;
+  status: number | null;
+  exitCode: number | null;
+  reason: CodexSteerFailureReason | null;
+  scope: ClassifiedCodexSteerError["scope"] | null;
+  retryable: boolean;
+  message: string | null;
+  upstreamBodyPreview?: unknown;
+}
+
 interface CompactTaskMutationResponseOptions {
  fullPrompt?: boolean;
 }
@@ -497,7 +516,10 @@ function responseBody(response: Record<string, unknown> | null): Record<string,
 function responseErrorMessage(response: Record<string, unknown> | null): string {
  const body = responseBody(response);
  const bodyError = body?.error;
-  if (typeof bodyError === "string" && bodyError.length > 0) return bodyError;
+  const providerError = body?.providerError;
+  if (typeof bodyError === "string" && bodyError.length > 0) {
+    return typeof providerError === "string" && providerError.length > 0 ? `${bodyError}: ${providerError}` : bodyError;
+  }
  if (typeof response?.codeQueueObservationNote === "string") return response.codeQueueObservationNote;
  if (typeof response?.stderrTail === "string" && response.stderrTail.length > 0) return response.stderrTail;
  if (typeof response?.commandStderrTail === "string" && response.commandStderrTail.length > 0) return response.commandStderrTail;
@@ -577,6 +599,41 @@ function unwrapSteerResponse(response: unknown, targetPath: string, stableProxyP
  return { ok: false, diagnostics: classifySteerFailure(response, targetPath, stableProxyPath, rawProxyEquivalent) };
 }

+function steerSuccessAttempt(attempt: number, durationMs: number, upstream: { ok: unknown; status: unknown }): CodexSteerAttemptSummary {
+  const status = typeof upstream.status === "number" && Number.isFinite(upstream.status) ? upstream.status : null;
+  return {
+    attempt,
+    ok: true,
+    durationMs,
+    status,
+    exitCode: null,
+    reason: null,
+    scope: null,
+    retryable: false,
+    message: "steer accepted by Code Queue",
+  };
+}
+
+function steerFailureAttempt(attempt: number, durationMs: number, diagnostics: ClassifiedCodexSteerError): CodexSteerAttemptSummary {
+  return {
+    attempt,
+    ok: false,
+    durationMs,
+    status: diagnostics.status,
+    exitCode: diagnostics.exitCode,
+    reason: diagnostics.reason,
+    scope: diagnostics.scope,
+    retryable: diagnostics.retryable,
+    message: diagnostics.message,
+    upstreamBodyPreview: diagnostics.upstreamBodyPreview,
+  };
+}
+
+function shouldRetrySteerFailure(diagnostics: ClassifiedCodexSteerError, attempt: number, maxAttempts: number): boolean {
+  if (attempt >= maxAttempts) return false;
+  return diagnostics.retryable === true && (diagnostics.reason === "stable-proxy-failed" || diagnostics.reason === "backend-core-unreachable");
+}
+
 function positiveIntegerOption(args: string[], names: string[], defaultValue: number, maxValue = Number.MAX_SAFE_INTEGER): number {
  for (const name of names) {
    const index = args.indexOf(name);
@@ -2705,6 +2762,8 @@ const submitPromptValueOptions = new Set([
 const steerPromptValueOptions = new Set([
  "--prompt-file",
  "--file",
+  "--retry-attempts",
+  "--retry-delay-ms",
 ]);

 function referenceTaskIdsFromOptions(args: string[]): string[] {
@@ -2758,12 +2817,17 @@ function parseSubmitOptions(args: string[]): CodexSubmitOptions {

 function parseSteerOptions(args: string[]): CodexSteerOptions {
  assertKnownOptions(args, {
-    flags: ["--prompt-stdin", "--stdin", "--dry-run"],
-    valueOptions: ["--prompt-file", "--file"],
+    flags: ["--prompt-stdin", "--stdin", "--dry-run", "--no-retry"],
+    valueOptions: ["--prompt-file", "--file", "--retry-attempts", "--retry-delay-ms"],
  }, "codex steer");
+  const retryAttempts = hasFlag(args, "--no-retry")
+    ? 1
+    : positiveIntegerOption(args, ["--retry-attempts"], defaultSteerRetryAttempts, maxSteerRetryAttempts);
  return {
    prompt: promptFromArgs(args, "codex steer", steerPromptValueOptions),
    dryRun: hasFlag(args, "--dry-run"),
+    retryAttempts,
+    retryDelayMs: nonNegativeIntegerOption(args, ["--retry-delay-ms"], defaultSteerRetryDelayMs, maxSteerRetryDelayMs),
  };
 }

@@ -3887,6 +3951,13 @@ function codexSteerTask(taskId: string, args: string[], fetcher: CodexResponseFe
    path: targetPath,
    stableProxyPath,
    method: "POST",
+    retryPolicy: {
+      defaultAttempts: defaultSteerRetryAttempts,
+      maxAttempts: options.retryAttempts,
+      delayMs: options.retryDelayMs,
+      retryableReasons: ["stable-proxy-failed", "backend-core-unreachable"],
+      deliveryConfirmation: "success confirms Code Queue accepted the steer request; repeated stable-proxy failures mean delivery is unconfirmed",
+    },
    bodySummary: {
      promptChars: options.prompt.length,
      promptPreviewChars: steerPromptPreviewChars,
@@ -3901,22 +3972,70 @@ function codexSteerTask(taskId: string, args: string[], fetcher: CodexResponseFe
      request,
      commands: {
        run: `bun scripts/cli.ts codex steer ${taskId} --prompt-file <path>`,
+        noRetry: `bun scripts/cli.ts codex steer ${taskId} --prompt-file <path> --no-retry`,
        rawProxy: rawProxyEquivalent,
      },
    };
  }
-  const response = unwrapSteerResponse(fetcher(stableProxyPath, { method: "POST", body: { prompt: options.prompt } }), targetPath, stableProxyPath, rawProxyEquivalent);
-  if (!response.ok) {
+  const attempts: CodexSteerAttemptSummary[] = [];
+  let failedResponse: { ok: false; diagnostics: ClassifiedCodexSteerError } | null = null;
+  let successfulResponse: { ok: true; upstream: { ok: unknown; status: unknown }; body: Record<string, unknown> } | null = null;
+  for (let attempt = 1; attempt <= options.retryAttempts; attempt += 1) {
+    const startedAt = Date.now();
+    const response = unwrapSteerResponse(fetcher(stableProxyPath, { method: "POST", body: { prompt: options.prompt } }), targetPath, stableProxyPath, rawProxyEquivalent);
+    const durationMs = Date.now() - startedAt;
+    if (response.ok) {
+      attempts.push(steerSuccessAttempt(attempt, durationMs, response.upstream));
+      successfulResponse = response;
+      break;
+    }
+    attempts.push(steerFailureAttempt(attempt, durationMs, response.diagnostics));
+    failedResponse = response;
+    if (!shouldRetrySteerFailure(response.diagnostics, attempt, options.retryAttempts)) break;
+    sleepSync(options.retryDelayMs);
+  }
+  if (successfulResponse === null) {
+    const diagnostics = failedResponse?.diagnostics ?? classifySteerFailure(null, targetPath, stableProxyPath, rawProxyEquivalent);
+    const transportDeliveryUnconfirmed = diagnostics.reason === "stable-proxy-failed" || diagnostics.reason === "backend-core-unreachable";
    return {
      ok: false,
      steer: {
        accepted: false,
        prompt,
+        attempts,
+        deliveryUnconfirmed: transportDeliveryUnconfirmed,
+        retryPolicy: {
+          attempted: attempts.length,
+          maxAttempts: options.retryAttempts,
+          retryDelayMs: options.retryDelayMs,
+          retried: attempts.length > 1,
+          exhausted: transportDeliveryUnconfirmed && attempts.length >= options.retryAttempts,
+          note: "codex steer retried only retryable stable-proxy/backend-core failures; raw microservice proxy uses the same tunnel and is diagnostic, not a lower-noise fallback.",
+        },
      },
      request,
-      diagnostics: response.diagnostics,
+      diagnostics: {
+        ...diagnostics,
+        attempts,
+        retryPolicy: {
+          attempted: attempts.length,
+          maxAttempts: options.retryAttempts,
+          retryDelayMs: options.retryDelayMs,
+          retried: attempts.length > 1,
+          exhausted: transportDeliveryUnconfirmed && attempts.length >= options.retryAttempts,
+        },
+        operatorGuidance: {
+          rawProxyEquivalentIsFallback: false,
+          deliveryUnconfirmed: transportDeliveryUnconfirmed,
+          nextStep: transportDeliveryUnconfirmed
+            ? "Check task liveness and retry codex steer from the main-server CLI or explicit SSH transport; do not treat a raw proxy failure as separate evidence that the task rejected the correction."
+            : "Inspect the non-retryable reason before resubmitting the correction.",
+        },
+      },
      commands: {
        dryRun: `bun scripts/cli.ts codex steer ${taskId} --prompt-file <path> --dry-run`,
+        retry: `bun scripts/cli.ts codex steer ${taskId} --prompt-file <path>`,
+        noRetry: `bun scripts/cli.ts codex steer ${taskId} --prompt-file <path> --no-retry`,
        rawProxy: rawProxyEquivalent,
        tasks: "bun scripts/cli.ts codex tasks --view supervisor --limit 20",
        health: "bun scripts/cli.ts microservice health code-queue",
@@ -3925,12 +4044,19 @@ function codexSteerTask(taskId: string, args: string[], fetcher: CodexResponseFe
  }
  return {
    ok: true,
-    upstream: response.upstream,
+    upstream: successfulResponse.upstream,
    steer: {
      accepted: true,
      taskId,
      promptChars: options.prompt.length,
      promptOmitted: true,
+      attempts,
+      retryPolicy: {
+        attempted: attempts.length,
+        maxAttempts: options.retryAttempts,
+        retryDelayMs: options.retryDelayMs,
+        retried: attempts.length > 1,
+      },
      outputPolicy: {
        default: "write-confirmation",
        promptEchoed: false,
@@ -3938,8 +4064,8 @@ function codexSteerTask(taskId: string, args: string[], fetcher: CodexResponseFe
        reason: "codex steer is a write operation; default output confirms delivery and provides drill-down commands without echoing prompt text or full task state.",
      },
    },
-    task: compactSubmitTaskConfirmation(response.body.task),
-    queue: compactSubmitQueueConfirmation(response.body.queue),
+    task: compactSubmitTaskConfirmation(successfulResponse.body.task),
+    queue: compactSubmitQueueConfirmation(successfulResponse.body.queue),
    commands: {
      show: `bun scripts/cli.ts codex task ${taskId}`,
      detail: `bun scripts/cli.ts codex task ${taskId} --detail`,