diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index 3b0a2ab6..464012a0 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -147,7 +147,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm - 上游报 capacity/rate-limit/overload/Bad Gateway/Gateway Timeout 后没有切号或频繁先失败再恢复：先确认 `codex-pool validate` 里 `tempUnschedulable.ok=true` 且目标 account `runtimeEnabled=true`、规则数符合 YAML；再看 `validation.gatewayResponses.evidence.failovers` 的 account/upstream status。若 mismatch，跑 `codex-pool sync --confirm`，不要手工 patch Sub2API credentials。 - Codex 报 weekly-limit、`less than 10% of your weekly limit left`、`Run /status for a breakdown` 等账号状态/软配额提示并要求切号：把稳定 body 关键词放进 `pool.defaultTempUnschedulable` 的 403 和 429 规则，跑 `codex-pool sync --confirm`，再用 `codex-pool validate` 确认每个 managed account 的 runtime 403/429 rules 都包含这些关键词。Sub2API 临时下线规则按 HTTP status + body keyword 匹配；如果该文案是 HTTP 200 成功内容，需要另提响应分类能力 issue，不能只靠 YAML 冷却规则声明解决。 - 上游 503 响应体出现 `model_not_found`、`No available channel for model ...` 或同类稳定模型路由失败文案：把稳定 body 关键词放进 `pool.defaultTempUnschedulable` 的 503 规则，跑 `codex-pool sync --confirm`，再用 `codex-pool validate` 确认目标 account 的 runtime 503 rule 包含这些关键词；不要用 account membership、priority、capacity、loadFactor、WebSocket mode 或 User-Agent 改动掩盖该错误族。 -- 上游错误反复触发：默认错误冷却按严重程度分层；临时问题可从 10 分钟起步，网关/服务不可用/过载/模型路由类应更长，认证/权限/配额/账号状态类使用最长冷却。`Recovered upstream error ...`、`Bad Gateway`、`Gateway Timeout`、Codex-facing `Upstream request failed`、`Unknown error`、`context deadline exceeded`、`context canceled`、`model_not_found`、`No available channel for model`、大上下文 `413` 和 `openai_error` 这类稳定包装文案都应留在 YAML 冷却政策里。具体数值只以 YAML 为准，修改后必须 `codex-pool sync --confirm` 和 `codex-pool validate`。长期判定见 `docs/reference/platform-infra.md`。 +- 上游错误反复触发：默认错误冷却按严重程度分层；临时问题可从 10 分钟起步，网关/服务不可用/过载/模型路由类应更长，认证/权限/配额/账号状态类使用最长冷却。`Recovered upstream error ...`、`Bad Gateway`、`Gateway Timeout`、Cloudflare `524`、Codex-facing `Upstream request failed`、`Unknown error`、`context deadline exceeded`、`context canceled`、`model_not_found`、`No available channel for model`、大上下文 `413` 和 `openai_error` 这类稳定包装文案都应留在 YAML 冷却政策里。具体数值只以 YAML 为准，修改后必须 `codex-pool sync --confirm` 和 `codex-pool validate`。长期判定见 `docs/reference/platform-infra.md`。 - Codex auto compact 后丢上下文：先确认本机 `~/.codex/config.toml` 是否有 `supports_websockets = true` 和 `responses_websockets_v2 = true`，再看 `codex-pool validate` 的 WSv2 candidate 和 Sub2API 日志里的 `transport=responses_websockets_v2`。 - Codex smoke 有 reconnect/1013：这是上游并发/可用性问题，和 HTTP-only compact context-loss 分开处理；记录 session/log 证据并关联专项 issue，不要用运行时手补覆盖 YAML 容量。 diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 6f1087b1..2ef98d8a 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -42,6 +42,10 @@ pool: keywords: [gateway timeout, timeout, upstream, upstream request failed, unknown error, context deadline exceeded, context canceled, recovered upstream error] durationMinutes: 30 description: Gateway timeout responses should cool down the selected account so another account can handle the next request. + - statusCode: 524 + keywords: [timeout, a timeout occurred, cloudflare, gateway timeout, upstream, context deadline exceeded] + durationMinutes: 30 + description: Cloudflare 524 timeout responses should cool down the selected account so another account can handle the next request. - statusCode: 529 keywords: [capacity, overloaded, temporarily unavailable, temporary, recovered upstream error] durationMinutes: 30 diff --git a/scripts/platform-infra-sub2api-codex-routing-contract-test.ts b/scripts/platform-infra-sub2api-codex-routing-contract-test.ts index f31651a8..81276b6c 100644 --- a/scripts/platform-infra-sub2api-codex-routing-contract-test.ts +++ b/scripts/platform-infra-sub2api-codex-routing-contract-test.ts @@ -58,6 +58,11 @@ if (parsed.pool?.defaultTempUnschedulable?.enabled === true) { for (const keyword of ["gateway timeout", "unknown error", "context deadline exceeded"]) { assertCondition(gateway504Keywords.has(keyword), "504 temporary-unschedulable rule must catch gateway timeout wrappers", { keyword, gateway504Rule }); } + const cloudflare524Rule = rules.find((rule) => rule.statusCode === 524); + const cloudflare524Keywords = new Set((cloudflare524Rule?.keywords ?? []).map((keyword) => keyword.toLowerCase())); + for (const keyword of ["timeout", "a timeout occurred", "cloudflare"]) { + assertCondition(cloudflare524Keywords.has(keyword), "524 temporary-unschedulable rule must catch Cloudflare timeout wrappers", { keyword, cloudflare524Rule }); + } const accountState403Rule = rules.find((rule) => rule.statusCode === 403); const quota429Rule = rules.find((rule) => rule.statusCode === 429); const serviceUnavailable503Rule = rules.find((rule) => rule.statusCode === 503); @@ -85,6 +90,7 @@ console.log(JSON.stringify({ "generic recovered upstream error wrappers are caught by cooldown rules", "large-context upstream failures are caught by the 413 cooldown rule", "gateway timeout wrappers are caught by the 504 cooldown rule", + "Cloudflare timeout wrappers are caught by the 524 cooldown rule", "Codex weekly-limit prompts are caught by account-state and quota cooldown rules", "upstream model-routing failures are caught by the 503 cooldown rule", "Responses smoke model is YAML-declared", diff --git a/scripts/platform-infra-sub2api-codex-temp-unsched-contract-test.ts b/scripts/platform-infra-sub2api-codex-temp-unsched-contract-test.ts index 841e9b9b..43f1a376 100644 --- a/scripts/platform-infra-sub2api-codex-temp-unsched-contract-test.ts +++ b/scripts/platform-infra-sub2api-codex-temp-unsched-contract-test.ts @@ -32,6 +32,7 @@ const quota429Rule = rules.find((rule) => rule.error_code === 429); const serviceUnavailable503Rule = rules.find((rule) => rule.error_code === 503); const gatewayTimeout504Rule = rules.find((rule) => rule.error_code === 504); const largeContext413Rule = rules.find((rule) => rule.error_code === 413); +const cloudflare524Rule = rules.find((rule) => rule.error_code === 524); for (const keyword of ["weekly limit", "less than 10% of your weekly limit left", "run /status for a breakdown"]) { assertCondition(accountState403Rule?.keywords?.includes(keyword), "403 rendered rule must preserve Codex weekly-limit account-state keyword", { keyword, accountState403Rule }); assertCondition(quota429Rule?.keywords?.includes(keyword), "429 rendered rule must preserve Codex weekly-limit quota keyword", { keyword, quota429Rule }); @@ -45,6 +46,9 @@ for (const keyword of ["openai_error", "context length", "maximum context"]) { for (const keyword of ["gateway timeout", "unknown error", "context deadline exceeded"]) { assertCondition(gatewayTimeout504Rule?.keywords?.includes(keyword), "504 rendered rule must preserve gateway-timeout cooldown keyword", { keyword, gatewayTimeout504Rule }); } +for (const keyword of ["timeout", "a timeout occurred", "cloudflare"]) { + assertCondition(cloudflare524Rule?.keywords?.includes(keyword), "524 rendered rule must catch Cloudflare timeout wrappers", { keyword, cloudflare524Rule }); +} const disabled = renderSub2ApiTempUnschedulableCredentials({ enabled: false, rules: policy.rules }) as { temp_unschedulable_enabled?: boolean; @@ -63,6 +67,7 @@ console.log(JSON.stringify({ "large-context upstream failures render into the 413 cooldown rule", "upstream model-routing failures render into the 503 cooldown rule", "gateway timeout wrappers render into the 504 cooldown rule", + "Cloudflare timeout wrappers render into the 524 cooldown rule", "disabled policies clear runtime rules", ], })); diff --git a/scripts/src/platform-infra-sub2api-codex.ts b/scripts/src/platform-infra-sub2api-codex.ts index 6c81a726..43bbe73c 100644 --- a/scripts/src/platform-infra-sub2api-codex.ts +++ b/scripts/src/platform-infra-sub2api-codex.ts @@ -700,6 +700,12 @@ export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePol durationMinutes: 30, description: "Gateway timeout responses should cool down the selected account so another account can handle the next request.", }, + { + statusCode: 524, + keywords: ["timeout", "a timeout occurred", "cloudflare", "gateway timeout", "upstream", "context deadline exceeded"], + durationMinutes: 30, + description: "Cloudflare 524 timeout responses should cool down the selected account so another account can handle the next request.", + }, { statusCode: 529, keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "recovered upstream error"],