fix: let recovered upstream error probes trigger cooldown
This commit is contained in:
@@ -45,6 +45,9 @@ if (parsed.pool?.defaultTempUnschedulable?.enabled === true) {
|
||||
assertCondition(rules.every((rule) => Number.isInteger(rule.statusCode) && (rule.statusCode ?? 0) >= 100 && (rule.statusCode ?? 0) <= 599), "temporary unschedulable rules must declare valid HTTP status codes", rules);
|
||||
assertCondition(rules.every((rule) => Array.isArray(rule.keywords) && rule.keywords.length > 0), "temporary unschedulable rules must declare non-empty keywords", rules);
|
||||
assertCondition(rules.every((rule) => Number.isInteger(rule.durationMinutes) && (rule.durationMinutes ?? 0) > 0), "temporary unschedulable rules must declare positive cooldown durations", rules);
|
||||
const gateway502Rule = rules.find((rule) => rule.statusCode === 502);
|
||||
const gateway502Keywords = new Set((gateway502Rule?.keywords ?? []).map((keyword) => keyword.toLowerCase()));
|
||||
assertCondition(gateway502Keywords.has("recovered upstream error"), "502 temporary-unschedulable rule must catch recovered upstream error wrappers", gateway502Rule);
|
||||
}
|
||||
assertCondition(typeof parsed.localCodex?.responsesSmokeModel === "string" && parsed.localCodex.responsesSmokeModel.length > 0, "localCodex.responsesSmokeModel must be declared for Responses smoke validation", parsed.localCodex);
|
||||
|
||||
@@ -56,6 +59,7 @@ console.log(JSON.stringify({
|
||||
"profile load factor overrides are YAML-controlled positive integers",
|
||||
"optional WebSocket mode overrides use supported values",
|
||||
"temporary unschedulable rules are structurally valid when enabled",
|
||||
"generic recovered upstream error wrappers are caught by cooldown rules",
|
||||
"Responses smoke model is YAML-declared",
|
||||
],
|
||||
}));
|
||||
|
||||
@@ -654,43 +654,43 @@ export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePol
|
||||
rules: [
|
||||
{
|
||||
statusCode: 401,
|
||||
keywords: ["unauthorized", "invalid api key", "invalid_api_key", "authentication"],
|
||||
keywords: ["unauthorized", "invalid api key", "invalid_api_key", "authentication", "recovered upstream error"],
|
||||
durationMinutes: 120,
|
||||
description: "Credential/auth failures should leave the scheduler quickly and retry after a two-hour cooldown.",
|
||||
description: "Credential/auth failures should use the longest cooldown.",
|
||||
},
|
||||
{
|
||||
statusCode: 403,
|
||||
keywords: ["forbidden", "access denied", "quota", "billing", "capacity"],
|
||||
keywords: ["forbidden", "access denied", "quota", "billing", "capacity", "recovered upstream error"],
|
||||
durationMinutes: 120,
|
||||
description: "Permission, quota, or account-state failures should fail over to another account for at least two hours.",
|
||||
description: "Permission, quota, or account-state failures should use the longest cooldown.",
|
||||
},
|
||||
{
|
||||
statusCode: 429,
|
||||
keywords: ["capacity", "rate limit", "rate_limit", "quota", "too many requests", "overloaded", "resource_exhausted"],
|
||||
keywords: ["capacity", "rate limit", "rate_limit", "quota", "too many requests", "overloaded", "resource_exhausted", "recovered upstream error"],
|
||||
durationMinutes: 10,
|
||||
description: "Capacity and rate-limit responses are often temporary; start with a ten-minute cooldown and use another account.",
|
||||
},
|
||||
{
|
||||
statusCode: 500,
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream"],
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "recovered upstream error"],
|
||||
durationMinutes: 10,
|
||||
description: "Transient upstream server failures should start with a ten-minute cooldown and prefer another account.",
|
||||
},
|
||||
{
|
||||
statusCode: 502,
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "bad gateway", "upstream request failed", "websocket dial", "handshake response"],
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "bad gateway", "upstream request failed", "websocket dial", "handshake response", "recovered upstream error"],
|
||||
durationMinutes: 30,
|
||||
description: "Gateway upstream failures are more disruptive than transient capacity signals and should cool down longer.",
|
||||
description: "Gateway upstream failures, including recovered upstream error wrappers, should cool down longer.",
|
||||
},
|
||||
{
|
||||
statusCode: 503,
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream"],
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "recovered upstream error"],
|
||||
durationMinutes: 30,
|
||||
description: "Service unavailable responses should cool down longer than one-off transient failures.",
|
||||
},
|
||||
{
|
||||
statusCode: 529,
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary"],
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "recovered upstream error"],
|
||||
durationMinutes: 30,
|
||||
description: "Provider overloaded responses should cool down longer than generic transient failures and use another account.",
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user