Merge pull request #96 from pikasTech/code-queue/issue-20-runner-429-backoff

fix: add Code Queue 429 backoff diagnostics

docs/reference/code-queue-supervision.md

@@ -252,7 +252,7 @@ bun scripts/cli.ts codex pr-preflight --remote --issue <issue-number>
 - Code Queue scheduler heartbeat、任务 heartbeat、trace/output 是否持续入库；
 - 当前 runner 容器内 CLI/proxy 路径是否只是局部不可达。
 
-只有多个独立观察面同时失败，或同一关键路径在明确时间窗口内持续失败，才能把问题判为全局阻塞。否则应记录为 transient、`runner-local-observation-gap` 或 `control-plane-observation-gap`，优先重试、steer 任务纠偏或拆出基础设施 follow-up；不得让业务 worker 把单次局部失败作为最终 blocker。CLI 和 runtime 必须把错误输出结构化为 `scope=runner-local|control-plane|provider-gateway|ssh|registry|k3s|scheduler|service-proxy`、`observedAt`、`retryable`、`decision` 或 `blockingDisposition`、`healthyScopes`、`failedScopes` 和建议的交叉验证命令。当前 runner/local backend-core 容器缺失属于 runner-local observation gap；远程控制面也不可达属于 control-plane observation gap；两者都不能单独写成 active runner 数归零或 scheduler 停摆。
+只有多个独立观察面同时失败，或同一关键路径在明确时间窗口内持续失败，才能把问题判为全局阻塞。否则应记录为 transient、`runner-local-observation-gap`、`external-provider-backoff` 或 `control-plane-observation-gap`，优先重试、steer 任务纠偏或拆出基础设施 follow-up；不得让业务 worker 把单次局部失败作为最终 blocker。CLI 和 runtime 必须把错误输出结构化为 `scope=runner-local|external-provider|control-plane|provider-gateway|ssh|registry|k3s|scheduler|service-proxy`、`observedAt`、`retryable`、`decision` 或 `blockingDisposition`、`healthyScopes`、`failedScopes` 和建议的交叉验证命令。当前 runner/local backend-core 容器缺失属于 runner-local observation gap；远程控制面也不可达属于 control-plane observation gap；两者都不能单独写成 active runner 数归零或 scheduler 停摆。
 
 ClaudeQQ 是面向用户的主动提醒通道，不是 #24 简报更新的自动转发器。指挥官只应在三类情况下自主发送 ClaudeQQ 消息：核心服务或关键执行面宕机且需要用户知情，高风险决策需要用户请示，或出现里程碑式进展值得同步。消息必须简明扼要，一次不超过 200 个中文字符，写成一段话，不使用 Markdown 语法。普通轮询、普通 issue 更新、普通 #24 简报追加、外部 token provider 正常限流、以及无用户动作要求的中间状态，不发送 ClaudeQQ。发送失败只记录到 #24 或对应 blocker issue，不回滚已经完成的 GitHub issue 更新。
 
@@ -268,7 +268,7 @@ D601 artifact registry 的 systemd unit inactive 不等于 D601 全局离线。
 
 对于 trace 或 heartbeat 新鲜的长任务，通常应保持运行。每几分钟轮询一次优于反复 interrupt/retry。
 
-外部 token provider、模型 API 或上游服务的限流和短时不可用是正常预期，不应自动升级为 Code Queue 基础设施缺陷。典型表现包括 `429 Too Many Requests`、provider transient error、上游 timeout 或模型服务短时失败。只要 Code Queue 的状态机仍在自动指数退避，task heartbeat 或 scheduler heartbeat 新鲜，且任务仍能从 `retry_wait` 回到 `running`，指挥官应等待外部 provider 自行恢复，不创建额外修复 issue、不重派重复任务、不把该现象写成 blocker。只有当退避机制失效、任务丢失、heartbeat 过期、状态机卡死，或重试耗尽进入不可恢复终态时，才按 Code Queue 基础设施问题介入。
+外部 token provider、模型 API 或上游服务的限流和短时不可用是正常预期，不应自动升级为 Code Queue 基础设施缺陷。典型表现包括 `429 Too Many Requests`、provider transient error、上游 timeout 或模型服务短时失败。runner 必须把这类 OpenAI/模型 provider 429 归类为 `scope=external-provider`、`failureKind=external-provider-rate-limit`、`externalProvider429=true`，并在 attempt 的 `runnerErrorClassification.backoffHint`、任务 output 的 `queue/backoff` 行和日志 `task_retry_backoff` 中暴露指数退避与 jitter 证据。退避策略是保守指数退避加稳定正向 jitter：429 至少等待 30 秒，单次不超过 10 分钟，jitter 按 task id 和 completed attempt count 稳定计算，避免多 runner 同时恢复造成 provider 再次拥塞。只要 Code Queue 的状态机仍在自动退避，task heartbeat 或 scheduler heartbeat 新鲜，且任务仍能从 `retry_wait` 回到 `running`，指挥官应等待外部 provider 自行恢复，不创建额外修复 issue、不重派重复任务、不把该现象写成 blocker。只有当退避机制失效、任务丢失、heartbeat 过期、状态机卡死，或重试耗尽进入不可恢复终态时，才按 Code Queue 基础设施问题介入。
 
 对于大规模 CI/CD 迁移波次，除非发生事故，否则使用稳定但可自适应的监督节奏。指挥官可以根据任务活跃度、完成未读积压、heartbeat 风险和外部等待性质自行决定 sleep 时长，但单次 sleep 不能低于 5 分钟、不能高于 30 分钟。活跃排障、刚派出新任务、存在完成未读或 heartbeat 风险时使用接近 5 分钟的短轮询；长时间等待外部 CI、模型 provider 退避或镜像构建且 heartbeat 新鲜时，可以拉长到 10 到 30 分钟。每轮醒来后读取 `codex queues`，读取 terminal 或可疑任务摘要，然后决定接受、retry、拆分 blocker，或让健康任务继续运行。循环期间指挥官可以做不重叠的有用工作，例如文档或 issue 梳理，但这些辅助工作不能接管 worker 已分配的实现。
 

scripts/code-queue-submit-routing-contract-test.ts

@@ -69,6 +69,9 @@ export function runCodeQueueSubmitRoutingContract(): JsonRecord {
   assertCondition(Array.isArray(modelTiers), "policy contract should expose model tiers", policyContract);
   const deepseekTier = modelTiers.map(asRecord).find((tier) => tier.model === "deepseek-chat");
   assertCondition(deepseekTier?.runner === "opencode", "DeepSeek policy tier should use OpenCode", policyContract);
+  const externalProvider429 = asRecord(policyContract.externalProvider429);
+  assertCondition(externalProvider429.commanderAction === "wait-while-exponential-backoff-is-healthy", "429 policy should tell commander to wait on healthy backoff", policyContract);
+  assertCondition(Array.isArray(externalProvider429.interveneWhen), "429 policy should expose intervention conditions", policyContract);
 
   const registry = codexSubmitModelRegistryForTest(["gpt-5.5", "deepseek", "minimax-m2.7"]);
   const modelPorts = asRecord(registry.modelPorts);

scripts/provider-runner-triage-contract-test.ts

@@ -41,6 +41,10 @@ export function runProviderRunnerTriageContract(): JsonRecord {
   assertScope("kubectl get pods failed: k3s api unavailable", "k3s", "service-degraded");
   assertScope("code-queue scheduler active run heartbeat is stale", "scheduler", "retryable-transient");
   assertScope("unexpected runner process exited with code 1", "unknown", "retryable-transient");
+  const rateLimit = assertScope("exceeded retry limit, last status: 429 Too Many Requests, request id: 21zqfw7apcg", "external-provider", "retryable-transient");
+  assertCondition(rateLimit.externalProvider429 === true, "OpenAI/model 429 should be explicit externalProvider429 evidence", rateLimit);
+  assertCondition(rateLimit.failureKind === "external-provider-rate-limit", "429 should use a stable rate-limit failure kind", rateLimit);
+  assertCondition((rateLimit.backoffHint as JsonRecord | undefined)?.strategy === "exponential-jitter", "429 classification should expose jittered backoff hint", rateLimit);
 
   const singlePath = classifyRunnerError("provider is not online: D601", "D601");
   const result = buildProviderTriageResult("D601", [
@@ -55,6 +59,14 @@ export function runProviderRunnerTriageContract(): JsonRecord {
   assertCondition(result.retryable === true, "single path provider offline should be retryable", result);
   assertCondition(result.contract.singlePathProviderOfflineIsGlobalBlocker === false, "triage contract should reject single-path global blocker", result);
 
+  const rateLimitTriage = buildProviderTriageResult("D601", [
+    signal("observed-runner-429", "external-provider", "failed", false),
+    signal("code-queue-health", "scheduler", "ok"),
+  ], "2026-05-21T00:00:00.000Z");
+  assertCondition(rateLimitTriage.blockingDisposition === "external-provider-backoff", "external provider 429 should stay in backoff disposition", rateLimitTriage);
+  assertCondition(rateLimitTriage.decision === "retryable-transient", "external provider 429 should remain retryable transient", rateLimitTriage);
+  assertCondition(rateLimitTriage.retryable === true, "external provider 429 should be retryable", rateLimitTriage);
+
   const cliSummary = codexTaskQuery("codex_runner_triage_fixture", ["--detail"], (path) => {
     assertCondition(path.includes("/api/microservices/code-queue/proxy/api/tasks/codex_runner_triage_fixture/summary"), "task summary should use stable proxy path", { path });
     return {
@@ -90,6 +102,7 @@ export function runProviderRunnerTriageContract(): JsonRecord {
       "runner error classifier separates runner-local/provider-gateway/registry/k3s/scheduler/unknown",
       "each single runner error classification has globalBlocker=false",
       "provider triage keeps single provider is not online as retryable-transient, not global-blocker",
+      "external OpenAI/model provider 429 is explicit retryable backoff evidence, not Code Queue infra outage",
       "codex task --detail preserves runnerErrorClassification in compact attempt output",
     ],
   };

scripts/src/provider-triage.ts

@@ -12,6 +12,7 @@ export type ProviderSignalScope =
   | "registry"
   | "k3s"
   | "scheduler"
+  | "external-provider"
   | "service-proxy"
   | "microservice"
   | "unknown";
@@ -21,6 +22,7 @@ export type ProviderSignalStatus = "ok" | "degraded" | "failed" | "unknown";
 export type ProviderBlockingDisposition =
   | "transient"
   | "runner-local-observation-gap"
+  | "external-provider-backoff"
   | "provider-degraded"
   | "service-degraded"
   | "global-blocker";
@@ -280,6 +282,7 @@ function codeQueueTasksSignal(response: unknown): ProviderTriageSignal {
 
 function classifyErrorMessage(message: string): ProviderSignalScope {
   const runnerClassification = classifyRunnerError(message);
+  if (runnerClassification.scope === "external-provider") return "external-provider";
   if (runnerClassification.scope === "provider-gateway") return "provider-gateway";
   if (runnerClassification.scope === "registry") return "registry";
   if (runnerClassification.scope === "k3s") return "k3s";
@@ -460,12 +463,16 @@ export function classifyProviderTriage(providerId: string, signals: ProviderTria
   const independentDegradedScopes = uniqueScopes(signals, ["degraded"]);
   const failedCriticalScopes = independentFailedScopes.filter((scope) => criticalScopes.has(scope));
   const runnerLocalObservedFailure = signals.some((signal) => signal.scope === "runner-local" && signal.status === "failed");
-  const serviceOnlyFailure = independentFailedScopes.length > 0 && independentFailedScopes.every((scope) => scope === "registry" || scope === "service-proxy" || scope === "microservice" || scope === "k3s");
+  const externalProviderObservedFailure = signals.some((signal) => signal.scope === "external-provider" && signal.status === "failed");
+  const serviceOnlyFailure = independentFailedScopes.length > 0 && independentFailedScopes.every((scope) => scope === "registry" || scope === "service-proxy" || scope === "microservice" || scope === "k3s" || scope === "external-provider");
   const hasIndependentHealthy = healthyScopes.length > 0;
   const rationale: string[] = [];
   let blockingDisposition: ProviderBlockingDisposition;
 
-  if (runnerLocalObservedFailure && independentFailedScopes.length === 0) {
+  if (externalProviderObservedFailure && failedCriticalScopes.length === 0) {
+    blockingDisposition = "external-provider-backoff";
+    rationale.push("external model provider 429/rate-limit should stay in Code Queue retry_wait with conservative backoff while scheduler heartbeat remains healthy");
+  } else if (runnerLocalObservedFailure && independentFailedScopes.length === 0) {
     blockingDisposition = "runner-local-observation-gap";
     rationale.push("single runner-local provider offline observation is not sufficient evidence for global D601 outage");
   } else if (failedCriticalScopes.length >= 2 && healthyScopes.length === 0) {

src/components/microservices/code-queue/src/index.ts

@@ -3435,11 +3435,94 @@ function retryBackoffMs(completedAttempts: number): number {
   return Math.min(retryBackoffMaxMs, retryBackoffBaseMs * (2 ** exponent));
 }
 
+function stableRetryJitterMs(taskId: string, completedAttempts: number, jitterWindowMs: number): number {
+  const boundedWindow = Math.max(0, Math.floor(jitterWindowMs));
+  if (boundedWindow <= 0) return 0;
+  let hash = 2166136261;
+  const input = `${taskId}:${Math.max(1, Math.floor(completedAttempts))}`;
+  for (let index = 0; index < input.length; index += 1) {
+    hash ^= input.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return (hash >>> 0) % (boundedWindow + 1);
+}
+
+function retryBackoffWithJitterMs(task: QueueTask, completedAttempts: number, minimumDelayMs = 0): { baseDelayMs: number; jitterMs: number; delayMs: number } {
+  const baseDelayMs = Math.min(retryBackoffMaxMs, Math.max(minimumDelayMs, retryBackoffMs(completedAttempts)));
+  const jitterWindowMs = Math.min(30_000, Math.max(1_000, Math.floor(baseDelayMs * 0.3)));
+  const jitterMs = stableRetryJitterMs(task.id, completedAttempts, jitterWindowMs);
+  return {
+    baseDelayMs,
+    jitterMs,
+    delayMs: Math.min(retryBackoffMaxMs, baseDelayMs + jitterMs),
+  };
+}
+
 function judgeProviderBackoffMs(task: QueueTask): number {
   const fallbackCount = Math.max(1, fallbackJudgeRetryCount(task));
   return Math.min(5 * 60_000, Math.max(30_000, retryBackoffMs(fallbackCount)));
 }
 
+function latestAttemptRunnerErrorClassification(task: QueueTask): Record<string, JsonValue> | null {
+  const latest = task.attempts.at(-1)?.runnerErrorClassification;
+  return typeof latest === "object" && latest !== null && !Array.isArray(latest) ? latest as Record<string, JsonValue> : null;
+}
+
+function latestAttemptExternalProvider429(task: QueueTask): boolean {
+  return latestAttemptRunnerErrorClassification(task)?.externalProvider429 === true;
+}
+
+function retryBackoffEvidence(task: QueueTask, completedAttempts: number, reason: string): JsonValue {
+  const runnerClassification = latestAttemptRunnerErrorClassification(task);
+  const externalProvider429 = runnerClassification?.externalProvider429 === true;
+  const baseDelayMs = retryBackoffMs(completedAttempts);
+  const jittered = externalProvider429
+    ? retryBackoffWithJitterMs(task, completedAttempts, 30_000)
+    : { baseDelayMs, jitterMs: 0, delayMs: baseDelayMs };
+  return {
+    strategy: externalProvider429 ? "exponential-jitter" : "exponential",
+    reason,
+    completedAttempts,
+    baseDelayMs: jittered.baseDelayMs,
+    jitterMs: jittered.jitterMs,
+    delayMs: jittered.delayMs,
+    maxDelayMs: retryBackoffMaxMs,
+    externalProvider429,
+    failureKind: runnerClassification?.failureKind ?? null,
+    runnerDisposition: runnerClassification?.disposition ?? null,
+    commanderBenefit: "fewer failed runners and clearer retry/backoff evidence",
+  };
+}
+
+function judgeProviderBackoffEvidence(task: QueueTask): JsonValue {
+  const fallbackCount = Math.max(1, fallbackJudgeRetryCount(task));
+  const baseDelayMs = judgeProviderBackoffMs(task);
+  const jitterMs = stableRetryJitterMs(task.id, fallbackCount, Math.min(30_000, Math.max(1_000, Math.floor(baseDelayMs * 0.3))));
+  return {
+    strategy: "exponential-jitter",
+    reason: "judge-provider-transient-retry",
+    completedAttempts: task.attempts.length,
+    baseDelayMs,
+    jitterMs,
+    delayMs: Math.min(5 * 60_000, baseDelayMs + jitterMs),
+    maxDelayMs: 5 * 60_000,
+    externalProvider429: false,
+    failureKind: "judge-provider-transient",
+    runnerDisposition: "retryable-transient",
+    commanderBenefit: "fewer failed runners and clearer retry/backoff evidence",
+  };
+}
+
+function appendRetryBackoffEvidence(task: QueueTask, evidence: JsonValue, continuation: string): void {
+  const record = typeof evidence === "object" && evidence !== null && !Array.isArray(evidence) ? evidence as Record<string, JsonValue> : {};
+  const delaySeconds = Math.round(Number(record.delayMs ?? 0) / 1000);
+  const baseSeconds = Math.round(Number(record.baseDelayMs ?? 0) / 1000);
+  const jitterSeconds = Math.round(Number(record.jitterMs ?? 0) / 1000);
+  const strategy = String(record.strategy ?? "exponential");
+  const external = record.externalProvider429 === true ? " externalProvider429=true" : "";
+  appendOutput(task, "system", `retry backoff ${delaySeconds}s (base=${baseSeconds}s jitter=${jitterSeconds}s strategy=${strategy}${external}) before ${continuation}\n`, "queue/backoff");
+}
+
 async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, message: string): Promise<T> {
   let timer: ReturnType<typeof setTimeout> | null = null;
   const timeout = new Promise<never>((_, reject) => {
@@ -3785,9 +3868,10 @@ async function runTask(task: QueueTask): Promise<void> {
           judgeFailRetryLimit,
           reason: safePreview(judge.reason, 500),
         });
-        const delayMs = retryBackoffMs(task.attempts.length);
-        appendOutput(task, "system", `retry backoff ${Math.round(delayMs / 1000)}s before appending continuation to existing session\n`, "queue");
-        await sleepForRetryBackoff(task, delayMs);
+        const backoff = retryBackoffEvidence(task, task.attempts.length, "judge-fail-treated-as-retry");
+        appendRetryBackoffEvidence(task, backoff, "appending continuation to existing session");
+        logger("info", "task_retry_backoff", { taskId: task.id, attempt: task.currentAttempt, backoff });
+        await sleepForRetryBackoff(task, Number((backoff as Record<string, JsonValue>).delayMs ?? 0));
         continue;
       }
       task.status = "failed";
@@ -3809,17 +3893,19 @@ async function runTask(task: QueueTask): Promise<void> {
       task.nextMode = "retry";
       setAttemptFeedbackPrompt(latestAttempt, task.nextPrompt, "judge-provider-transient-retry", task.attempts.length + 1);
       task.updatedAt = nowIso();
-      const delayMs = judgeProviderBackoffMs(task);
-      appendOutput(task, "system", `judge provider transient failure; retry backoff ${Math.round(delayMs / 1000)}s before continuing existing session\n`, "queue");
+      const backoff = judgeProviderBackoffEvidence(task);
+      const providerDelayMs = Number((backoff as Record<string, JsonValue>).delayMs ?? 0);
+      appendRetryBackoffEvidence(task, backoff, "continuing existing session after judge provider transient failure");
       persistTaskState(task);
       await flushDirtyTasksToDatabase(true);
       logger("warn", "task_judge_provider_transient_retry", {
         taskId: task.id,
         fallbackRetryCount: fallbackJudgeRetryCount(task),
-        delayMs,
+        delayMs: providerDelayMs,
+        backoff,
         reason: safePreview(judge.reason, 500),
       });
-      await sleepForRetryBackoff(task, delayMs);
+      await sleepForRetryBackoff(task, providerDelayMs);
       continue;
     }
     if (judge.source === "fallback" && fallbackJudgeRetryCount(task) >= fallbackJudgeRetryLimit) {
@@ -3837,9 +3923,11 @@ async function runTask(task: QueueTask): Promise<void> {
     task.updatedAt = nowIso();
     persistTaskState(task);
     await flushDirtyTasksToDatabase(true);
-    const delayMs = retryBackoffMs(task.attempts.length);
-    appendOutput(task, "system", `retry backoff ${Math.round(delayMs / 1000)}s before appending continuation to existing session\n`, "queue");
-    await sleepForRetryBackoff(task, delayMs);
+    const externalProvider429 = latestAttemptExternalProvider429(task);
+    const backoff = retryBackoffEvidence(task, task.attempts.length, externalProvider429 ? "external-provider-429" : "judge-retry");
+    appendRetryBackoffEvidence(task, backoff, "appending continuation to existing session");
+    logger(externalProvider429 ? "warn" : "info", "task_retry_backoff", { taskId: task.id, attempt: task.currentAttempt, backoff });
+    await sleepForRetryBackoff(task, Number((backoff as Record<string, JsonValue>).delayMs ?? 0));
   }
 
   if (shutdownRequested) {

src/components/microservices/code-queue/src/runner-error-classifier.ts

@@ -6,6 +6,7 @@ export type RunnerFailureScope =
   | "registry"
   | "k3s"
   | "scheduler"
+  | "external-provider"
   | "unknown";
 
 export type RunnerFailureDisposition =
@@ -18,15 +19,25 @@ export type RunnerFailureDisposition =
 export interface RunnerErrorClassification {
   scope: RunnerFailureScope;
   disposition: RunnerFailureDisposition;
+  failureKind: string;
   retryable: boolean;
   globalBlocker: boolean;
   singlePathObservation: boolean;
+  externalProvider429: boolean;
   reason: string;
   evidence: string[];
   recommendedTriageCommand: string;
+  backoffHint: {
+    strategy: "exponential-jitter";
+    minimumDelayMs: number;
+    maxDelayMs: number;
+    jitter: string;
+    commanderAction: string;
+  } | null;
   contract: {
     singlePathFailureIsGlobalBlocker: false;
     globalBlockerRequiresMultiSignalTriage: true;
+    externalProvider429IsInfrastructureOutage: false;
   };
 }
 
@@ -95,6 +106,17 @@ export function classifyRunnerError(message: string, providerId?: string | null)
     /postgres|postgresql/gu,
     /app-server closed|opencode exited|codex .*failed/gu,
   ]);
+  const externalProvider429Evidence = collectEvidence(normalized, [
+    /429 too many requests/gu,
+    /too many requests/gu,
+    /\bstatus:?\s*429\b/gu,
+    /\bhttp\s*429\b/gu,
+    /\b429\b/gu,
+    /rate[- ]?limit(?:ed|ing)?/gu,
+    /quota exceeded/gu,
+    /exceeded retry limit/gu,
+    /tokens per min|requests per min/gu,
+  ]);
   const runnerLocalEvidence = collectEvidence(normalized, [
     /spawn .*enoent/gu,
     /command not found/gu,
@@ -107,37 +129,52 @@ export function classifyRunnerError(message: string, providerId?: string | null)
 
   let scope: RunnerFailureScope = "unknown";
   let disposition: RunnerFailureDisposition = "retryable-transient";
+  let failureKind = "unknown-runner-failure";
   let reason = "The runner failure needs provider triage before it can be treated as a global blocker.";
   let evidence: string[] = [];
+  let externalProvider429 = false;
 
-  if (providerOfflineEvidence.length > 0) {
+  if (externalProvider429Evidence.length > 0) {
+    scope = "external-provider";
+    disposition = "retryable-transient";
+    failureKind = "external-provider-rate-limit";
+    externalProvider429 = true;
+    reason = "The failure matches an external model provider rate limit/transient 429; keep the task in retry_wait with conservative jittered backoff while scheduler heartbeat remains fresh.";
+    evidence = externalProvider429Evidence;
+  } else if (providerOfflineEvidence.length > 0) {
     scope = "runner-local";
     disposition = "runner-local-observation-gap";
+    failureKind = "provider-offline-single-path";
     reason = "A single runner-local provider offline observation is not independent evidence of a global D601/provider outage.";
     evidence = providerOfflineEvidence;
   } else if (registryEvidence.length > 0) {
     scope = "registry";
     disposition = "service-degraded";
+    failureKind = "registry-path-failure";
     reason = "The failure is registry-scoped; cross-check provider gateway, SSH, k3s and scheduler before escalating globally.";
     evidence = registryEvidence;
   } else if (k3sEvidence.length > 0) {
     scope = "k3s";
     disposition = "service-degraded";
+    failureKind = "k3s-path-failure";
     reason = "The failure is k3s-scoped unless provider gateway and SSH cross-checks also fail.";
     evidence = k3sEvidence;
   } else if (providerGatewayEvidence.length > 0) {
     scope = "provider-gateway";
     disposition = "provider-degraded";
+    failureKind = "provider-gateway-path-failure";
     reason = "The failure points at provider-gateway/proxy transport and requires independent SSH/scheduler checks.";
     evidence = providerGatewayEvidence;
   } else if (schedulerEvidence.length > 0) {
     scope = "scheduler";
     disposition = "retryable-transient";
+    failureKind = "scheduler-runner-failure";
     reason = "The failure is scheduler/runner scoped and should be retried or triaged with runtime preflight.";
     evidence = schedulerEvidence;
   } else if (runnerLocalEvidence.length > 0) {
     scope = "runner-local";
     disposition = "runner-local-observation-gap";
+    failureKind = "runner-local-observation";
     reason = "The failure was observed locally by the runner and is not enough to declare provider-global outage.";
     evidence = runnerLocalEvidence;
   }
@@ -145,15 +182,27 @@ export function classifyRunnerError(message: string, providerId?: string | null)
   return {
     scope,
     disposition,
+    failureKind,
     retryable: true,
     globalBlocker: false,
     singlePathObservation: true,
+    externalProvider429,
     reason,
     evidence,
     recommendedTriageCommand: providerTriageCommand(providerId, raw, scope),
+    backoffHint: externalProvider429
+      ? {
+        strategy: "exponential-jitter",
+        minimumDelayMs: 30_000,
+        maxDelayMs: 10 * 60 * 1000,
+        jitter: "stable positive jitter keyed by task id and completed attempt count",
+        commanderAction: "wait for retry_wait to return to running while heartbeat remains fresh; do not duplicate the task unless retry attempts exhaust or heartbeat/state machine stalls",
+      }
+      : null,
     contract: {
       singlePathFailureIsGlobalBlocker: false,
       globalBlockerRequiresMultiSignalTriage: true,
+      externalProvider429IsInfrastructureOutage: false,
     },
   };
 }