Merge pull request #96 from pikasTech/code-queue/issue-20-runner-429-backoff

fix: add Code Queue 429 backoff diagnostics

scripts/code-queue-submit-routing-contract-test.ts

@@ -69,6 +69,9 @@ export function runCodeQueueSubmitRoutingContract(): JsonRecord {
   assertCondition(Array.isArray(modelTiers), "policy contract should expose model tiers", policyContract);
   const deepseekTier = modelTiers.map(asRecord).find((tier) => tier.model === "deepseek-chat");
   assertCondition(deepseekTier?.runner === "opencode", "DeepSeek policy tier should use OpenCode", policyContract);
+  const externalProvider429 = asRecord(policyContract.externalProvider429);
+  assertCondition(externalProvider429.commanderAction === "wait-while-exponential-backoff-is-healthy", "429 policy should tell commander to wait on healthy backoff", policyContract);
+  assertCondition(Array.isArray(externalProvider429.interveneWhen), "429 policy should expose intervention conditions", policyContract);
 
   const registry = codexSubmitModelRegistryForTest(["gpt-5.5", "deepseek", "minimax-m2.7"]);
   const modelPorts = asRecord(registry.modelPorts);

scripts/provider-runner-triage-contract-test.ts

@@ -41,6 +41,10 @@ export function runProviderRunnerTriageContract(): JsonRecord {
   assertScope("kubectl get pods failed: k3s api unavailable", "k3s", "service-degraded");
   assertScope("code-queue scheduler active run heartbeat is stale", "scheduler", "retryable-transient");
   assertScope("unexpected runner process exited with code 1", "unknown", "retryable-transient");
+  const rateLimit = assertScope("exceeded retry limit, last status: 429 Too Many Requests, request id: 21zqfw7apcg", "external-provider", "retryable-transient");
+  assertCondition(rateLimit.externalProvider429 === true, "OpenAI/model 429 should be explicit externalProvider429 evidence", rateLimit);
+  assertCondition(rateLimit.failureKind === "external-provider-rate-limit", "429 should use a stable rate-limit failure kind", rateLimit);
+  assertCondition((rateLimit.backoffHint as JsonRecord | undefined)?.strategy === "exponential-jitter", "429 classification should expose jittered backoff hint", rateLimit);
 
   const singlePath = classifyRunnerError("provider is not online: D601", "D601");
   const result = buildProviderTriageResult("D601", [
@@ -55,6 +59,14 @@ export function runProviderRunnerTriageContract(): JsonRecord {
   assertCondition(result.retryable === true, "single path provider offline should be retryable", result);
   assertCondition(result.contract.singlePathProviderOfflineIsGlobalBlocker === false, "triage contract should reject single-path global blocker", result);
 
+  const rateLimitTriage = buildProviderTriageResult("D601", [
+    signal("observed-runner-429", "external-provider", "failed", false),
+    signal("code-queue-health", "scheduler", "ok"),
+  ], "2026-05-21T00:00:00.000Z");
+  assertCondition(rateLimitTriage.blockingDisposition === "external-provider-backoff", "external provider 429 should stay in backoff disposition", rateLimitTriage);
+  assertCondition(rateLimitTriage.decision === "retryable-transient", "external provider 429 should remain retryable transient", rateLimitTriage);
+  assertCondition(rateLimitTriage.retryable === true, "external provider 429 should be retryable", rateLimitTriage);
+
   const cliSummary = codexTaskQuery("codex_runner_triage_fixture", ["--detail"], (path) => {
     assertCondition(path.includes("/api/microservices/code-queue/proxy/api/tasks/codex_runner_triage_fixture/summary"), "task summary should use stable proxy path", { path });
     return {
@@ -90,6 +102,7 @@ export function runProviderRunnerTriageContract(): JsonRecord {
       "runner error classifier separates runner-local/provider-gateway/registry/k3s/scheduler/unknown",
       "each single runner error classification has globalBlocker=false",
       "provider triage keeps single provider is not online as retryable-transient, not global-blocker",
+      "external OpenAI/model provider 429 is explicit retryable backoff evidence, not Code Queue infra outage",
       "codex task --detail preserves runnerErrorClassification in compact attempt output",
     ],
   };

scripts/src/provider-triage.ts

@@ -12,6 +12,7 @@ export type ProviderSignalScope =
   | "registry"
   | "k3s"
   | "scheduler"
+  | "external-provider"
   | "service-proxy"
   | "microservice"
   | "unknown";
@@ -21,6 +22,7 @@ export type ProviderSignalStatus = "ok" | "degraded" | "failed" | "unknown";
 export type ProviderBlockingDisposition =
   | "transient"
   | "runner-local-observation-gap"
+  | "external-provider-backoff"
   | "provider-degraded"
   | "service-degraded"
   | "global-blocker";
@@ -280,6 +282,7 @@ function codeQueueTasksSignal(response: unknown): ProviderTriageSignal {
 
 function classifyErrorMessage(message: string): ProviderSignalScope {
   const runnerClassification = classifyRunnerError(message);
+  if (runnerClassification.scope === "external-provider") return "external-provider";
   if (runnerClassification.scope === "provider-gateway") return "provider-gateway";
   if (runnerClassification.scope === "registry") return "registry";
   if (runnerClassification.scope === "k3s") return "k3s";
@@ -460,12 +463,16 @@ export function classifyProviderTriage(providerId: string, signals: ProviderTria
   const independentDegradedScopes = uniqueScopes(signals, ["degraded"]);
   const failedCriticalScopes = independentFailedScopes.filter((scope) => criticalScopes.has(scope));
   const runnerLocalObservedFailure = signals.some((signal) => signal.scope === "runner-local" && signal.status === "failed");
-  const serviceOnlyFailure = independentFailedScopes.length > 0 && independentFailedScopes.every((scope) => scope === "registry" || scope === "service-proxy" || scope === "microservice" || scope === "k3s");
+  const externalProviderObservedFailure = signals.some((signal) => signal.scope === "external-provider" && signal.status === "failed");
+  const serviceOnlyFailure = independentFailedScopes.length > 0 && independentFailedScopes.every((scope) => scope === "registry" || scope === "service-proxy" || scope === "microservice" || scope === "k3s" || scope === "external-provider");
   const hasIndependentHealthy = healthyScopes.length > 0;
   const rationale: string[] = [];
   let blockingDisposition: ProviderBlockingDisposition;
 
-  if (runnerLocalObservedFailure && independentFailedScopes.length === 0) {
+  if (externalProviderObservedFailure && failedCriticalScopes.length === 0) {
+    blockingDisposition = "external-provider-backoff";
+    rationale.push("external model provider 429/rate-limit should stay in Code Queue retry_wait with conservative backoff while scheduler heartbeat remains healthy");
+  } else if (runnerLocalObservedFailure && independentFailedScopes.length === 0) {
     blockingDisposition = "runner-local-observation-gap";
     rationale.push("single runner-local provider offline observation is not sufficient evidence for global D601 outage");
   } else if (failedCriticalScopes.length >= 2 && healthyScopes.length === 0) {