From 3d8305ea15d7a4c9a84624f882c241390210c4dd Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 23 May 2026 02:30:48 +0000 Subject: [PATCH] fix: add code queue 429 backoff diagnostics --- docs/reference/code-queue-supervision.md | 4 +- ...code-queue-submit-routing-contract-test.ts | 3 + .../provider-runner-triage-contract-test.ts | 13 +++ scripts/src/provider-triage.ts | 11 +- .../microservices/code-queue/src/index.ts | 108 ++++++++++++++++-- .../code-queue/src/runner-error-classifier.ts | 51 ++++++++- 6 files changed, 175 insertions(+), 15 deletions(-) diff --git a/docs/reference/code-queue-supervision.md b/docs/reference/code-queue-supervision.md index ff3761d6..ba403371 100644 --- a/docs/reference/code-queue-supervision.md +++ b/docs/reference/code-queue-supervision.md @@ -252,7 +252,7 @@ bun scripts/cli.ts codex pr-preflight --remote --issue - Code Queue scheduler heartbeat、任务 heartbeat、trace/output 是否持续入库； - 当前 runner 容器内 CLI/proxy 路径是否只是局部不可达。 -只有多个独立观察面同时失败，或同一关键路径在明确时间窗口内持续失败，才能把问题判为全局阻塞。否则应记录为 transient、`runner-local-observation-gap` 或 `control-plane-observation-gap`，优先重试、steer 任务纠偏或拆出基础设施 follow-up；不得让业务 worker 把单次局部失败作为最终 blocker。CLI 和 runtime 必须把错误输出结构化为 `scope=runner-local|control-plane|provider-gateway|ssh|registry|k3s|scheduler|service-proxy`、`observedAt`、`retryable`、`decision` 或 `blockingDisposition`、`healthyScopes`、`failedScopes` 和建议的交叉验证命令。当前 runner/local backend-core 容器缺失属于 runner-local observation gap；远程控制面也不可达属于 control-plane observation gap；两者都不能单独写成 active runner 数归零或 scheduler 停摆。 +只有多个独立观察面同时失败，或同一关键路径在明确时间窗口内持续失败，才能把问题判为全局阻塞。否则应记录为 transient、`runner-local-observation-gap`、`external-provider-backoff` 或 `control-plane-observation-gap`，优先重试、steer 任务纠偏或拆出基础设施 follow-up；不得让业务 worker 把单次局部失败作为最终 blocker。CLI 和 runtime 必须把错误输出结构化为 `scope=runner-local|external-provider|control-plane|provider-gateway|ssh|registry|k3s|scheduler|service-proxy`、`observedAt`、`retryable`、`decision` 或 `blockingDisposition`、`healthyScopes`、`failedScopes` 和建议的交叉验证命令。当前 runner/local backend-core 容器缺失属于 runner-local observation gap；远程控制面也不可达属于 control-plane observation gap；两者都不能单独写成 active runner 数归零或 scheduler 停摆。 ClaudeQQ 是面向用户的主动提醒通道，不是 #24 简报更新的自动转发器。指挥官只应在三类情况下自主发送 ClaudeQQ 消息：核心服务或关键执行面宕机且需要用户知情，高风险决策需要用户请示，或出现里程碑式进展值得同步。消息必须简明扼要，一次不超过 200 个中文字符，写成一段话，不使用 Markdown 语法。普通轮询、普通 issue 更新、普通 #24 简报追加、外部 token provider 正常限流、以及无用户动作要求的中间状态，不发送 ClaudeQQ。发送失败只记录到 #24 或对应 blocker issue，不回滚已经完成的 GitHub issue 更新。 @@ -268,7 +268,7 @@ D601 artifact registry 的 systemd unit inactive 不等于 D601 全局离线。 对于 trace 或 heartbeat 新鲜的长任务，通常应保持运行。每几分钟轮询一次优于反复 interrupt/retry。 -外部 token provider、模型 API 或上游服务的限流和短时不可用是正常预期，不应自动升级为 Code Queue 基础设施缺陷。典型表现包括 `429 Too Many Requests`、provider transient error、上游 timeout 或模型服务短时失败。只要 Code Queue 的状态机仍在自动指数退避，task heartbeat 或 scheduler heartbeat 新鲜，且任务仍能从 `retry_wait` 回到 `running`，指挥官应等待外部 provider 自行恢复，不创建额外修复 issue、不重派重复任务、不把该现象写成 blocker。只有当退避机制失效、任务丢失、heartbeat 过期、状态机卡死，或重试耗尽进入不可恢复终态时，才按 Code Queue 基础设施问题介入。 +外部 token provider、模型 API 或上游服务的限流和短时不可用是正常预期，不应自动升级为 Code Queue 基础设施缺陷。典型表现包括 `429 Too Many Requests`、provider transient error、上游 timeout 或模型服务短时失败。runner 必须把这类 OpenAI/模型 provider 429 归类为 `scope=external-provider`、`failureKind=external-provider-rate-limit`、`externalProvider429=true`，并在 attempt 的 `runnerErrorClassification.backoffHint`、任务 output 的 `queue/backoff` 行和日志 `task_retry_backoff` 中暴露指数退避与 jitter 证据。退避策略是保守指数退避加稳定正向 jitter：429 至少等待 30 秒，单次不超过 10 分钟，jitter 按 task id 和 completed attempt count 稳定计算，避免多 runner 同时恢复造成 provider 再次拥塞。只要 Code Queue 的状态机仍在自动退避，task heartbeat 或 scheduler heartbeat 新鲜，且任务仍能从 `retry_wait` 回到 `running`，指挥官应等待外部 provider 自行恢复，不创建额外修复 issue、不重派重复任务、不把该现象写成 blocker。只有当退避机制失效、任务丢失、heartbeat 过期、状态机卡死，或重试耗尽进入不可恢复终态时，才按 Code Queue 基础设施问题介入。 对于大规模 CI/CD 迁移波次，除非发生事故，否则使用稳定但可自适应的监督节奏。指挥官可以根据任务活跃度、完成未读积压、heartbeat 风险和外部等待性质自行决定 sleep 时长，但单次 sleep 不能低于 5 分钟、不能高于 30 分钟。活跃排障、刚派出新任务、存在完成未读或 heartbeat 风险时使用接近 5 分钟的短轮询；长时间等待外部 CI、模型 provider 退避或镜像构建且 heartbeat 新鲜时，可以拉长到 10 到 30 分钟。每轮醒来后读取 `codex queues`，读取 terminal 或可疑任务摘要，然后决定接受、retry、拆分 blocker，或让健康任务继续运行。循环期间指挥官可以做不重叠的有用工作，例如文档或 issue 梳理，但这些辅助工作不能接管 worker 已分配的实现。 diff --git a/scripts/code-queue-submit-routing-contract-test.ts b/scripts/code-queue-submit-routing-contract-test.ts index 9e02ad35..bcdfe85f 100644 --- a/scripts/code-queue-submit-routing-contract-test.ts +++ b/scripts/code-queue-submit-routing-contract-test.ts @@ -69,6 +69,9 @@ export function runCodeQueueSubmitRoutingContract(): JsonRecord { assertCondition(Array.isArray(modelTiers), "policy contract should expose model tiers", policyContract); const deepseekTier = modelTiers.map(asRecord).find((tier) => tier.model === "deepseek-chat"); assertCondition(deepseekTier?.runner === "opencode", "DeepSeek policy tier should use OpenCode", policyContract); + const externalProvider429 = asRecord(policyContract.externalProvider429); + assertCondition(externalProvider429.commanderAction === "wait-while-exponential-backoff-is-healthy", "429 policy should tell commander to wait on healthy backoff", policyContract); + assertCondition(Array.isArray(externalProvider429.interveneWhen), "429 policy should expose intervention conditions", policyContract); const registry = codexSubmitModelRegistryForTest(["gpt-5.5", "deepseek", "minimax-m2.7"]); const modelPorts = asRecord(registry.modelPorts); diff --git a/scripts/provider-runner-triage-contract-test.ts b/scripts/provider-runner-triage-contract-test.ts index e70fcdb7..10844bf5 100644 --- a/scripts/provider-runner-triage-contract-test.ts +++ b/scripts/provider-runner-triage-contract-test.ts @@ -41,6 +41,10 @@ export function runProviderRunnerTriageContract(): JsonRecord { assertScope("kubectl get pods failed: k3s api unavailable", "k3s", "service-degraded"); assertScope("code-queue scheduler active run heartbeat is stale", "scheduler", "retryable-transient"); assertScope("unexpected runner process exited with code 1", "unknown", "retryable-transient"); + const rateLimit = assertScope("exceeded retry limit, last status: 429 Too Many Requests, request id: 21zqfw7apcg", "external-provider", "retryable-transient"); + assertCondition(rateLimit.externalProvider429 === true, "OpenAI/model 429 should be explicit externalProvider429 evidence", rateLimit); + assertCondition(rateLimit.failureKind === "external-provider-rate-limit", "429 should use a stable rate-limit failure kind", rateLimit); + assertCondition((rateLimit.backoffHint as JsonRecord | undefined)?.strategy === "exponential-jitter", "429 classification should expose jittered backoff hint", rateLimit); const singlePath = classifyRunnerError("provider is not online: D601", "D601"); const result = buildProviderTriageResult("D601", [ @@ -55,6 +59,14 @@ export function runProviderRunnerTriageContract(): JsonRecord { assertCondition(result.retryable === true, "single path provider offline should be retryable", result); assertCondition(result.contract.singlePathProviderOfflineIsGlobalBlocker === false, "triage contract should reject single-path global blocker", result); + const rateLimitTriage = buildProviderTriageResult("D601", [ + signal("observed-runner-429", "external-provider", "failed", false), + signal("code-queue-health", "scheduler", "ok"), + ], "2026-05-21T00:00:00.000Z"); + assertCondition(rateLimitTriage.blockingDisposition === "external-provider-backoff", "external provider 429 should stay in backoff disposition", rateLimitTriage); + assertCondition(rateLimitTriage.decision === "retryable-transient", "external provider 429 should remain retryable transient", rateLimitTriage); + assertCondition(rateLimitTriage.retryable === true, "external provider 429 should be retryable", rateLimitTriage); + const cliSummary = codexTaskQuery("codex_runner_triage_fixture", ["--detail"], (path) => { assertCondition(path.includes("/api/microservices/code-queue/proxy/api/tasks/codex_runner_triage_fixture/summary"), "task summary should use stable proxy path", { path }); return { @@ -90,6 +102,7 @@ export function runProviderRunnerTriageContract(): JsonRecord { "runner error classifier separates runner-local/provider-gateway/registry/k3s/scheduler/unknown", "each single runner error classification has globalBlocker=false", "provider triage keeps single provider is not online as retryable-transient, not global-blocker", + "external OpenAI/model provider 429 is explicit retryable backoff evidence, not Code Queue infra outage", "codex task --detail preserves runnerErrorClassification in compact attempt output", ], }; diff --git a/scripts/src/provider-triage.ts b/scripts/src/provider-triage.ts index 8c188b98..a0b0dff2 100644 --- a/scripts/src/provider-triage.ts +++ b/scripts/src/provider-triage.ts @@ -12,6 +12,7 @@ export type ProviderSignalScope = | "registry" | "k3s" | "scheduler" + | "external-provider" | "service-proxy" | "microservice" | "unknown"; @@ -21,6 +22,7 @@ export type ProviderSignalStatus = "ok" | "degraded" | "failed" | "unknown"; export type ProviderBlockingDisposition = | "transient" | "runner-local-observation-gap" + | "external-provider-backoff" | "provider-degraded" | "service-degraded" | "global-blocker"; @@ -280,6 +282,7 @@ function codeQueueTasksSignal(response: unknown): ProviderTriageSignal { function classifyErrorMessage(message: string): ProviderSignalScope { const runnerClassification = classifyRunnerError(message); + if (runnerClassification.scope === "external-provider") return "external-provider"; if (runnerClassification.scope === "provider-gateway") return "provider-gateway"; if (runnerClassification.scope === "registry") return "registry"; if (runnerClassification.scope === "k3s") return "k3s"; @@ -460,12 +463,16 @@ export function classifyProviderTriage(providerId: string, signals: ProviderTria const independentDegradedScopes = uniqueScopes(signals, ["degraded"]); const failedCriticalScopes = independentFailedScopes.filter((scope) => criticalScopes.has(scope)); const runnerLocalObservedFailure = signals.some((signal) => signal.scope === "runner-local" && signal.status === "failed"); - const serviceOnlyFailure = independentFailedScopes.length > 0 && independentFailedScopes.every((scope) => scope === "registry" || scope === "service-proxy" || scope === "microservice" || scope === "k3s"); + const externalProviderObservedFailure = signals.some((signal) => signal.scope === "external-provider" && signal.status === "failed"); + const serviceOnlyFailure = independentFailedScopes.length > 0 && independentFailedScopes.every((scope) => scope === "registry" || scope === "service-proxy" || scope === "microservice" || scope === "k3s" || scope === "external-provider"); const hasIndependentHealthy = healthyScopes.length > 0; const rationale: string[] = []; let blockingDisposition: ProviderBlockingDisposition; - if (runnerLocalObservedFailure && independentFailedScopes.length === 0) { + if (externalProviderObservedFailure && failedCriticalScopes.length === 0) { + blockingDisposition = "external-provider-backoff"; + rationale.push("external model provider 429/rate-limit should stay in Code Queue retry_wait with conservative backoff while scheduler heartbeat remains healthy"); + } else if (runnerLocalObservedFailure && independentFailedScopes.length === 0) { blockingDisposition = "runner-local-observation-gap"; rationale.push("single runner-local provider offline observation is not sufficient evidence for global D601 outage"); } else if (failedCriticalScopes.length >= 2 && healthyScopes.length === 0) { diff --git a/src/components/microservices/code-queue/src/index.ts b/src/components/microservices/code-queue/src/index.ts index c4a35080..4b492527 100644 --- a/src/components/microservices/code-queue/src/index.ts +++ b/src/components/microservices/code-queue/src/index.ts @@ -3435,11 +3435,94 @@ function retryBackoffMs(completedAttempts: number): number { return Math.min(retryBackoffMaxMs, retryBackoffBaseMs * (2 ** exponent)); } +function stableRetryJitterMs(taskId: string, completedAttempts: number, jitterWindowMs: number): number { + const boundedWindow = Math.max(0, Math.floor(jitterWindowMs)); + if (boundedWindow <= 0) return 0; + let hash = 2166136261; + const input = `${taskId}:${Math.max(1, Math.floor(completedAttempts))}`; + for (let index = 0; index < input.length; index += 1) { + hash ^= input.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + return (hash >>> 0) % (boundedWindow + 1); +} + +function retryBackoffWithJitterMs(task: QueueTask, completedAttempts: number, minimumDelayMs = 0): { baseDelayMs: number; jitterMs: number; delayMs: number } { + const baseDelayMs = Math.min(retryBackoffMaxMs, Math.max(minimumDelayMs, retryBackoffMs(completedAttempts))); + const jitterWindowMs = Math.min(30_000, Math.max(1_000, Math.floor(baseDelayMs * 0.3))); + const jitterMs = stableRetryJitterMs(task.id, completedAttempts, jitterWindowMs); + return { + baseDelayMs, + jitterMs, + delayMs: Math.min(retryBackoffMaxMs, baseDelayMs + jitterMs), + }; +} + function judgeProviderBackoffMs(task: QueueTask): number { const fallbackCount = Math.max(1, fallbackJudgeRetryCount(task)); return Math.min(5 * 60_000, Math.max(30_000, retryBackoffMs(fallbackCount))); } +function latestAttemptRunnerErrorClassification(task: QueueTask): Record | null { + const latest = task.attempts.at(-1)?.runnerErrorClassification; + return typeof latest === "object" && latest !== null && !Array.isArray(latest) ? latest as Record : null; +} + +function latestAttemptExternalProvider429(task: QueueTask): boolean { + return latestAttemptRunnerErrorClassification(task)?.externalProvider429 === true; +} + +function retryBackoffEvidence(task: QueueTask, completedAttempts: number, reason: string): JsonValue { + const runnerClassification = latestAttemptRunnerErrorClassification(task); + const externalProvider429 = runnerClassification?.externalProvider429 === true; + const baseDelayMs = retryBackoffMs(completedAttempts); + const jittered = externalProvider429 + ? retryBackoffWithJitterMs(task, completedAttempts, 30_000) + : { baseDelayMs, jitterMs: 0, delayMs: baseDelayMs }; + return { + strategy: externalProvider429 ? "exponential-jitter" : "exponential", + reason, + completedAttempts, + baseDelayMs: jittered.baseDelayMs, + jitterMs: jittered.jitterMs, + delayMs: jittered.delayMs, + maxDelayMs: retryBackoffMaxMs, + externalProvider429, + failureKind: runnerClassification?.failureKind ?? null, + runnerDisposition: runnerClassification?.disposition ?? null, + commanderBenefit: "fewer failed runners and clearer retry/backoff evidence", + }; +} + +function judgeProviderBackoffEvidence(task: QueueTask): JsonValue { + const fallbackCount = Math.max(1, fallbackJudgeRetryCount(task)); + const baseDelayMs = judgeProviderBackoffMs(task); + const jitterMs = stableRetryJitterMs(task.id, fallbackCount, Math.min(30_000, Math.max(1_000, Math.floor(baseDelayMs * 0.3)))); + return { + strategy: "exponential-jitter", + reason: "judge-provider-transient-retry", + completedAttempts: task.attempts.length, + baseDelayMs, + jitterMs, + delayMs: Math.min(5 * 60_000, baseDelayMs + jitterMs), + maxDelayMs: 5 * 60_000, + externalProvider429: false, + failureKind: "judge-provider-transient", + runnerDisposition: "retryable-transient", + commanderBenefit: "fewer failed runners and clearer retry/backoff evidence", + }; +} + +function appendRetryBackoffEvidence(task: QueueTask, evidence: JsonValue, continuation: string): void { + const record = typeof evidence === "object" && evidence !== null && !Array.isArray(evidence) ? evidence as Record : {}; + const delaySeconds = Math.round(Number(record.delayMs ?? 0) / 1000); + const baseSeconds = Math.round(Number(record.baseDelayMs ?? 0) / 1000); + const jitterSeconds = Math.round(Number(record.jitterMs ?? 0) / 1000); + const strategy = String(record.strategy ?? "exponential"); + const external = record.externalProvider429 === true ? " externalProvider429=true" : ""; + appendOutput(task, "system", `retry backoff ${delaySeconds}s (base=${baseSeconds}s jitter=${jitterSeconds}s strategy=${strategy}${external}) before ${continuation}\n`, "queue/backoff"); +} + async function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { let timer: ReturnType | null = null; const timeout = new Promise((_, reject) => { @@ -3785,9 +3868,10 @@ async function runTask(task: QueueTask): Promise { judgeFailRetryLimit, reason: safePreview(judge.reason, 500), }); - const delayMs = retryBackoffMs(task.attempts.length); - appendOutput(task, "system", `retry backoff ${Math.round(delayMs / 1000)}s before appending continuation to existing session\n`, "queue"); - await sleepForRetryBackoff(task, delayMs); + const backoff = retryBackoffEvidence(task, task.attempts.length, "judge-fail-treated-as-retry"); + appendRetryBackoffEvidence(task, backoff, "appending continuation to existing session"); + logger("info", "task_retry_backoff", { taskId: task.id, attempt: task.currentAttempt, backoff }); + await sleepForRetryBackoff(task, Number((backoff as Record).delayMs ?? 0)); continue; } task.status = "failed"; @@ -3809,17 +3893,19 @@ async function runTask(task: QueueTask): Promise { task.nextMode = "retry"; setAttemptFeedbackPrompt(latestAttempt, task.nextPrompt, "judge-provider-transient-retry", task.attempts.length + 1); task.updatedAt = nowIso(); - const delayMs = judgeProviderBackoffMs(task); - appendOutput(task, "system", `judge provider transient failure; retry backoff ${Math.round(delayMs / 1000)}s before continuing existing session\n`, "queue"); + const backoff = judgeProviderBackoffEvidence(task); + const providerDelayMs = Number((backoff as Record).delayMs ?? 0); + appendRetryBackoffEvidence(task, backoff, "continuing existing session after judge provider transient failure"); persistTaskState(task); await flushDirtyTasksToDatabase(true); logger("warn", "task_judge_provider_transient_retry", { taskId: task.id, fallbackRetryCount: fallbackJudgeRetryCount(task), - delayMs, + delayMs: providerDelayMs, + backoff, reason: safePreview(judge.reason, 500), }); - await sleepForRetryBackoff(task, delayMs); + await sleepForRetryBackoff(task, providerDelayMs); continue; } if (judge.source === "fallback" && fallbackJudgeRetryCount(task) >= fallbackJudgeRetryLimit) { @@ -3837,9 +3923,11 @@ async function runTask(task: QueueTask): Promise { task.updatedAt = nowIso(); persistTaskState(task); await flushDirtyTasksToDatabase(true); - const delayMs = retryBackoffMs(task.attempts.length); - appendOutput(task, "system", `retry backoff ${Math.round(delayMs / 1000)}s before appending continuation to existing session\n`, "queue"); - await sleepForRetryBackoff(task, delayMs); + const externalProvider429 = latestAttemptExternalProvider429(task); + const backoff = retryBackoffEvidence(task, task.attempts.length, externalProvider429 ? "external-provider-429" : "judge-retry"); + appendRetryBackoffEvidence(task, backoff, "appending continuation to existing session"); + logger(externalProvider429 ? "warn" : "info", "task_retry_backoff", { taskId: task.id, attempt: task.currentAttempt, backoff }); + await sleepForRetryBackoff(task, Number((backoff as Record).delayMs ?? 0)); } if (shutdownRequested) { diff --git a/src/components/microservices/code-queue/src/runner-error-classifier.ts b/src/components/microservices/code-queue/src/runner-error-classifier.ts index 9b42b54c..0b69b2df 100644 --- a/src/components/microservices/code-queue/src/runner-error-classifier.ts +++ b/src/components/microservices/code-queue/src/runner-error-classifier.ts @@ -6,6 +6,7 @@ export type RunnerFailureScope = | "registry" | "k3s" | "scheduler" + | "external-provider" | "unknown"; export type RunnerFailureDisposition = @@ -18,15 +19,25 @@ export type RunnerFailureDisposition = export interface RunnerErrorClassification { scope: RunnerFailureScope; disposition: RunnerFailureDisposition; + failureKind: string; retryable: boolean; globalBlocker: boolean; singlePathObservation: boolean; + externalProvider429: boolean; reason: string; evidence: string[]; recommendedTriageCommand: string; + backoffHint: { + strategy: "exponential-jitter"; + minimumDelayMs: number; + maxDelayMs: number; + jitter: string; + commanderAction: string; + } | null; contract: { singlePathFailureIsGlobalBlocker: false; globalBlockerRequiresMultiSignalTriage: true; + externalProvider429IsInfrastructureOutage: false; }; } @@ -95,6 +106,17 @@ export function classifyRunnerError(message: string, providerId?: string | null) /postgres|postgresql/gu, /app-server closed|opencode exited|codex .*failed/gu, ]); + const externalProvider429Evidence = collectEvidence(normalized, [ + /429 too many requests/gu, + /too many requests/gu, + /\bstatus:?\s*429\b/gu, + /\bhttp\s*429\b/gu, + /\b429\b/gu, + /rate[- ]?limit(?:ed|ing)?/gu, + /quota exceeded/gu, + /exceeded retry limit/gu, + /tokens per min|requests per min/gu, + ]); const runnerLocalEvidence = collectEvidence(normalized, [ /spawn .*enoent/gu, /command not found/gu, @@ -107,37 +129,52 @@ export function classifyRunnerError(message: string, providerId?: string | null) let scope: RunnerFailureScope = "unknown"; let disposition: RunnerFailureDisposition = "retryable-transient"; + let failureKind = "unknown-runner-failure"; let reason = "The runner failure needs provider triage before it can be treated as a global blocker."; let evidence: string[] = []; + let externalProvider429 = false; - if (providerOfflineEvidence.length > 0) { + if (externalProvider429Evidence.length > 0) { + scope = "external-provider"; + disposition = "retryable-transient"; + failureKind = "external-provider-rate-limit"; + externalProvider429 = true; + reason = "The failure matches an external model provider rate limit/transient 429; keep the task in retry_wait with conservative jittered backoff while scheduler heartbeat remains fresh."; + evidence = externalProvider429Evidence; + } else if (providerOfflineEvidence.length > 0) { scope = "runner-local"; disposition = "runner-local-observation-gap"; + failureKind = "provider-offline-single-path"; reason = "A single runner-local provider offline observation is not independent evidence of a global D601/provider outage."; evidence = providerOfflineEvidence; } else if (registryEvidence.length > 0) { scope = "registry"; disposition = "service-degraded"; + failureKind = "registry-path-failure"; reason = "The failure is registry-scoped; cross-check provider gateway, SSH, k3s and scheduler before escalating globally."; evidence = registryEvidence; } else if (k3sEvidence.length > 0) { scope = "k3s"; disposition = "service-degraded"; + failureKind = "k3s-path-failure"; reason = "The failure is k3s-scoped unless provider gateway and SSH cross-checks also fail."; evidence = k3sEvidence; } else if (providerGatewayEvidence.length > 0) { scope = "provider-gateway"; disposition = "provider-degraded"; + failureKind = "provider-gateway-path-failure"; reason = "The failure points at provider-gateway/proxy transport and requires independent SSH/scheduler checks."; evidence = providerGatewayEvidence; } else if (schedulerEvidence.length > 0) { scope = "scheduler"; disposition = "retryable-transient"; + failureKind = "scheduler-runner-failure"; reason = "The failure is scheduler/runner scoped and should be retried or triaged with runtime preflight."; evidence = schedulerEvidence; } else if (runnerLocalEvidence.length > 0) { scope = "runner-local"; disposition = "runner-local-observation-gap"; + failureKind = "runner-local-observation"; reason = "The failure was observed locally by the runner and is not enough to declare provider-global outage."; evidence = runnerLocalEvidence; } @@ -145,15 +182,27 @@ export function classifyRunnerError(message: string, providerId?: string | null) return { scope, disposition, + failureKind, retryable: true, globalBlocker: false, singlePathObservation: true, + externalProvider429, reason, evidence, recommendedTriageCommand: providerTriageCommand(providerId, raw, scope), + backoffHint: externalProvider429 + ? { + strategy: "exponential-jitter", + minimumDelayMs: 30_000, + maxDelayMs: 10 * 60 * 1000, + jitter: "stable positive jitter keyed by task id and completed attempt count", + commanderAction: "wait for retry_wait to return to running while heartbeat remains fresh; do not duplicate the task unless retry attempts exhaust or heartbeat/state machine stalls", + } + : null, contract: { singlePathFailureIsGlobalBlocker: false, globalBlockerRequiresMultiSignalTriage: true, + externalProvider429IsInfrastructureOutage: false, }, }; }