Merge pull request #242 from pikasTech/fix/sub2api-524-cooldown-238
补齐 Sub2API 524 超时冷却规则
This commit is contained in:
Lyon
@@ -147,7 +147,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm
|
||||
- 上游报 capacity/rate-limit/overload/Bad Gateway/Gateway Timeout 后没有切号或频繁先失败再恢复：先确认 `codex-pool validate` 里 `tempUnschedulable.ok=true` 且目标 account `runtimeEnabled=true`、规则数符合 YAML；再看 `validation.gatewayResponses.evidence.failovers` 的 account/upstream status。若 mismatch，跑 `codex-pool sync --confirm`，不要手工 patch Sub2API credentials。
|
||||
- Codex 报 weekly-limit、`less than 10% of your weekly limit left`、`Run /status for a breakdown` 等账号状态/软配额提示并要求切号：把稳定 body 关键词放进 `pool.defaultTempUnschedulable` 的 403 和 429 规则，跑 `codex-pool sync --confirm`，再用 `codex-pool validate` 确认每个 managed account 的 runtime 403/429 rules 都包含这些关键词。Sub2API 临时下线规则按 HTTP status + body keyword 匹配；如果该文案是 HTTP 200 成功内容，需要另提响应分类能力 issue，不能只靠 YAML 冷却规则声明解决。
|
||||
- 上游 503 响应体出现 `model_not_found`、`No available channel for model ...` 或同类稳定模型路由失败文案：把稳定 body 关键词放进 `pool.defaultTempUnschedulable` 的 503 规则，跑 `codex-pool sync --confirm`，再用 `codex-pool validate` 确认目标 account 的 runtime 503 rule 包含这些关键词；不要用 account membership、priority、capacity、loadFactor、WebSocket mode 或 User-Agent 改动掩盖该错误族。
|
||||
- 上游错误反复触发：默认错误冷却按严重程度分层；临时问题可从 10 分钟起步，网关/服务不可用/过载/模型路由类应更长，认证/权限/配额/账号状态类使用最长冷却。`Recovered upstream error ...`、`Bad Gateway`、`Gateway Timeout`、Codex-facing `Upstream request failed`、`Unknown error`、`context deadline exceeded`、`context canceled`、`model_not_found`、`No available channel for model`、大上下文 `413` 和 `openai_error` 这类稳定包装文案都应留在 YAML 冷却政策里。具体数值只以 YAML 为准，修改后必须 `codex-pool sync --confirm` 和 `codex-pool validate`。长期判定见 `docs/reference/platform-infra.md`。
|
||||
- 上游错误反复触发：默认错误冷却按严重程度分层；临时问题可从 10 分钟起步，网关/服务不可用/过载/模型路由类应更长，认证/权限/配额/账号状态类使用最长冷却。`Recovered upstream error ...`、`Bad Gateway`、`Gateway Timeout`、Cloudflare `524`、Codex-facing `Upstream request failed`、`Unknown error`、`context deadline exceeded`、`context canceled`、`model_not_found`、`No available channel for model`、大上下文 `413` 和 `openai_error` 这类稳定包装文案都应留在 YAML 冷却政策里。具体数值只以 YAML 为准，修改后必须 `codex-pool sync --confirm` 和 `codex-pool validate`。长期判定见 `docs/reference/platform-infra.md`。
|
||||
- Codex auto compact 后丢上下文：先确认本机 `~/.codex/config.toml` 是否有 `supports_websockets = true` 和 `responses_websockets_v2 = true`，再看 `codex-pool validate` 的 WSv2 candidate 和 Sub2API 日志里的 `transport=responses_websockets_v2`。
|
||||
- Codex smoke 有 reconnect/1013：这是上游并发/可用性问题，和 HTTP-only compact context-loss 分开处理；记录 session/log 证据并关联专项 issue，不要用运行时手补覆盖 YAML 容量。
|
||||
|
||||
|
||||
@@ -42,6 +42,10 @@ pool:
|
||||
keywords: [gateway timeout, timeout, upstream, upstream request failed, unknown error, context deadline exceeded, context canceled, recovered upstream error]
|
||||
durationMinutes: 30
|
||||
description: Gateway timeout responses should cool down the selected account so another account can handle the next request.
|
||||
- statusCode: 524
|
||||
keywords: [timeout, a timeout occurred, cloudflare, gateway timeout, upstream, context deadline exceeded]
|
||||
durationMinutes: 30
|
||||
description: Cloudflare 524 timeout responses should cool down the selected account so another account can handle the next request.
|
||||
- statusCode: 529
|
||||
keywords: [capacity, overloaded, temporarily unavailable, temporary, recovered upstream error]
|
||||
durationMinutes: 30
|
||||
|
||||
@@ -58,6 +58,11 @@ if (parsed.pool?.defaultTempUnschedulable?.enabled === true) {
|
||||
for (const keyword of ["gateway timeout", "unknown error", "context deadline exceeded"]) {
|
||||
assertCondition(gateway504Keywords.has(keyword), "504 temporary-unschedulable rule must catch gateway timeout wrappers", { keyword, gateway504Rule });
|
||||
}
|
||||
const cloudflare524Rule = rules.find((rule) => rule.statusCode === 524);
|
||||
const cloudflare524Keywords = new Set((cloudflare524Rule?.keywords ?? []).map((keyword) => keyword.toLowerCase()));
|
||||
for (const keyword of ["timeout", "a timeout occurred", "cloudflare"]) {
|
||||
assertCondition(cloudflare524Keywords.has(keyword), "524 temporary-unschedulable rule must catch Cloudflare timeout wrappers", { keyword, cloudflare524Rule });
|
||||
}
|
||||
const accountState403Rule = rules.find((rule) => rule.statusCode === 403);
|
||||
const quota429Rule = rules.find((rule) => rule.statusCode === 429);
|
||||
const serviceUnavailable503Rule = rules.find((rule) => rule.statusCode === 503);
|
||||
@@ -85,6 +90,7 @@ console.log(JSON.stringify({
|
||||
"generic recovered upstream error wrappers are caught by cooldown rules",
|
||||
"large-context upstream failures are caught by the 413 cooldown rule",
|
||||
"gateway timeout wrappers are caught by the 504 cooldown rule",
|
||||
"Cloudflare timeout wrappers are caught by the 524 cooldown rule",
|
||||
"Codex weekly-limit prompts are caught by account-state and quota cooldown rules",
|
||||
"upstream model-routing failures are caught by the 503 cooldown rule",
|
||||
"Responses smoke model is YAML-declared",
|
||||
|
||||
@@ -32,6 +32,7 @@ const quota429Rule = rules.find((rule) => rule.error_code === 429);
|
||||
const serviceUnavailable503Rule = rules.find((rule) => rule.error_code === 503);
|
||||
const gatewayTimeout504Rule = rules.find((rule) => rule.error_code === 504);
|
||||
const largeContext413Rule = rules.find((rule) => rule.error_code === 413);
|
||||
const cloudflare524Rule = rules.find((rule) => rule.error_code === 524);
|
||||
for (const keyword of ["weekly limit", "less than 10% of your weekly limit left", "run /status for a breakdown"]) {
|
||||
assertCondition(accountState403Rule?.keywords?.includes(keyword), "403 rendered rule must preserve Codex weekly-limit account-state keyword", { keyword, accountState403Rule });
|
||||
assertCondition(quota429Rule?.keywords?.includes(keyword), "429 rendered rule must preserve Codex weekly-limit quota keyword", { keyword, quota429Rule });
|
||||
@@ -45,6 +46,9 @@ for (const keyword of ["openai_error", "context length", "maximum context"]) {
|
||||
for (const keyword of ["gateway timeout", "unknown error", "context deadline exceeded"]) {
|
||||
assertCondition(gatewayTimeout504Rule?.keywords?.includes(keyword), "504 rendered rule must preserve gateway-timeout cooldown keyword", { keyword, gatewayTimeout504Rule });
|
||||
}
|
||||
for (const keyword of ["timeout", "a timeout occurred", "cloudflare"]) {
|
||||
assertCondition(cloudflare524Rule?.keywords?.includes(keyword), "524 rendered rule must catch Cloudflare timeout wrappers", { keyword, cloudflare524Rule });
|
||||
}
|
||||
|
||||
const disabled = renderSub2ApiTempUnschedulableCredentials({ enabled: false, rules: policy.rules }) as {
|
||||
temp_unschedulable_enabled?: boolean;
|
||||
@@ -63,6 +67,7 @@ console.log(JSON.stringify({
|
||||
"large-context upstream failures render into the 413 cooldown rule",
|
||||
"upstream model-routing failures render into the 503 cooldown rule",
|
||||
"gateway timeout wrappers render into the 504 cooldown rule",
|
||||
"Cloudflare timeout wrappers render into the 524 cooldown rule",
|
||||
"disabled policies clear runtime rules",
|
||||
],
|
||||
}));
|
||||
|
||||
@@ -700,6 +700,12 @@ export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePol
|
||||
durationMinutes: 30,
|
||||
description: "Gateway timeout responses should cool down the selected account so another account can handle the next request.",
|
||||
},
|
||||
{
|
||||
statusCode: 524,
|
||||
keywords: ["timeout", "a timeout occurred", "cloudflare", "gateway timeout", "upstream", "context deadline exceeded"],
|
||||
durationMinutes: 30,
|
||||
description: "Cloudflare 524 timeout responses should cool down the selected account so another account can handle the next request.",
|
||||
},
|
||||
{
|
||||
statusCode: 529,
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "recovered upstream error"],
|
||||
|
||||
Reference in New Issue
Block a user