mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: add compact timeout cooldown evidence

Browse Source
This commit is contained in:
Codex
2026-06-10 07:42:21 +00:00
parent 9710a1a6f6
commit 8735b4103c
6 changed files with 85 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/platform-infra-sub2api-codex-routing-contract-test.ts
+4 -1
View File
@@ -59,6 +59,9 @@ if (parsed.pool?.defaultTempUnschedulable?.enabled === true) {
const gateway502Rule = rules.find((rule) => rule.statusCode === 502);
const gateway502Keywords = new Set((gateway502Rule?.keywords ?? []).map((keyword) => keyword.toLowerCase()));
assertCondition(gateway502Keywords.has("recovered upstream error"), "502 temporary-unschedulable rule must catch recovered upstream error wrappers", gateway502Rule);
for (const keyword of ["unknown error", "upstream request failed", "context canceled"]) {
assertCondition(gateway502Keywords.has(keyword), "502 temporary-unschedulable rule must catch compact gateway wrappers", { keyword, gateway502Rule });
}
const largeContext413Rule = rules.find((rule) => rule.statusCode === 413);
const largeContext413Keywords = new Set((largeContext413Rule?.keywords ?? []).map((keyword) => keyword.toLowerCase()));
for (const keyword of ["openai_error", "context length", "maximum context"]) {
@@ -71,7 +74,7 @@ if (parsed.pool?.defaultTempUnschedulable?.enabled === true) {
}
const cloudflare524Rule = rules.find((rule) => rule.statusCode === 524);
const cloudflare524Keywords = new Set((cloudflare524Rule?.keywords ?? []).map((keyword) => keyword.toLowerCase()));
for (const keyword of ["timeout", "a timeout occurred", "cloudflare"]) {
for (const keyword of ["timeout", "a timeout occurred", "cloudflare", "unknown error", "upstream request failed", "context canceled"]) {
assertCondition(cloudflare524Keywords.has(keyword), "524 temporary-unschedulable rule must catch Cloudflare timeout wrappers", { keyword, cloudflare524Rule });
}
const accountState403Rule = rules.find((rule) => rule.statusCode === 403);
scripts/platform-infra-sub2api-codex-temp-unsched-contract-test.ts
+5 -1
View File
@@ -29,6 +29,7 @@ assertCondition(!("pool_mode" in credentials), "pool_mode must not be enabled be
assertCondition(!("api_key" in credentials) && !("base_url" in credentials), "temporary-unschedulable rendering must not include secrets or endpoints", credentials);
const accountState403Rule = rules.find((rule) => rule.error_code === 403);
const quota429Rule = rules.find((rule) => rule.error_code === 429);
const gateway502Rule = rules.find((rule) => rule.error_code === 502);
const serviceUnavailable503Rule = rules.find((rule) => rule.error_code === 503);
const gatewayTimeout504Rule = rules.find((rule) => rule.error_code === 504);
const largeContext413Rule = rules.find((rule) => rule.error_code === 413);
@@ -43,10 +44,13 @@ for (const keyword of ["model_not_found", "no available channel for model"]) {
for (const keyword of ["openai_error", "context length", "maximum context"]) {
assertCondition(largeContext413Rule?.keywords?.includes(keyword), "413 rendered rule must catch large-context upstream failures", { keyword, largeContext413Rule });
}
for (const keyword of ["unknown error", "upstream request failed", "context canceled"]) {
assertCondition(gateway502Rule?.keywords?.includes(keyword), "502 rendered rule must catch compact gateway wrappers", { keyword, gateway502Rule });
}
for (const keyword of ["gateway timeout", "unknown error", "context deadline exceeded"]) {
assertCondition(gatewayTimeout504Rule?.keywords?.includes(keyword), "504 rendered rule must preserve gateway-timeout cooldown keyword", { keyword, gatewayTimeout504Rule });
}
for (const keyword of ["timeout", "a timeout occurred", "cloudflare"]) {
for (const keyword of ["timeout", "a timeout occurred", "cloudflare", "unknown error", "upstream request failed", "context canceled"]) {
assertCondition(cloudflare524Rule?.keywords?.includes(keyword), "524 rendered rule must catch Cloudflare timeout wrappers", { keyword, cloudflare524Rule });
}
scripts/src/platform-infra-sub2api-codex.ts
+71 -7
View File
@@ -694,7 +694,7 @@ export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePol
},
{
statusCode: 502,
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "bad gateway", "upstream request failed", "websocket dial", "handshake response", "recovered upstream error"],
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "bad gateway", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "websocket dial", "handshake response", "recovered upstream error"],
durationMinutes: 30,
description: "Gateway upstream failures, including recovered upstream error wrappers, should cool down longer.",
},
@@ -718,7 +718,7 @@ export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePol
},
{
statusCode: 524,
keywords: ["timeout", "a timeout occurred", "cloudflare", "gateway timeout", "upstream", "context deadline exceeded"],
keywords: ["timeout", "a timeout occurred", "cloudflare", "gateway timeout", "upstream", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "recovered upstream error"],
durationMinutes: 30,
description: "Cloudflare 524 timeout responses should cool down the selected account so another account can handle the next request.",
},
@@ -2326,6 +2326,68 @@ def request_log_evidence(request_id):
"logsStderr": text(proc.stderr, 1000),
}
def recent_compact_gateway_evidence():
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"])
stdout = proc.stdout.decode("utf-8", errors="replace")
failures = []
successes = []
failovers = []
final_errors = []
context_canceled = []
for line in stdout.splitlines():
if "/responses/compact" not in line and "remote_compact" not in line:
continue
json_start = line.find("{")
if json_start < 0:
continue
try:
item = json.loads(line[json_start:])
except Exception:
continue
path = item.get("path")
entry = {
"requestId": item.get("request_id"),
"clientRequestId": item.get("client_request_id"),
"accountId": item.get("account_id"),
"statusCode": item.get("status_code"),
"upstreamStatus": item.get("upstream_status"),
"latencyMs": item.get("latency_ms"),
"path": path,
}
if "codex.remote_compact.failed" in line:
failures.append(entry)
elif "codex.remote_compact.succeeded" in line:
successes.append(entry)
elif "upstream_failover_switching" in line and path == "/responses/compact":
failovers.append({
**entry,
"switchCount": item.get("switch_count"),
"maxSwitches": item.get("max_switches"),
})
elif "http request completed" in line and path == "/responses/compact" and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
final_errors.append(entry)
if "context canceled" in line and path == "/responses/compact":
context_canceled.append(entry)
return {
"ok": True,
"degraded": len(failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0,
"window": "6h",
"tailLines": 2500,
"failureCount": len(failures),
"successCount": len(successes),
"failoverCount": len(failovers),
"finalErrorCount": len(final_errors),
"contextCanceledCount": len(context_canceled),
"recentFailures": failures[-5:],
"recentSuccesses": successes[-5:],
"recentFailovers": failovers[-8:],
"recentFinalErrors": final_errors[-5:],
"recentContextCanceled": context_canceled[-5:],
"logsExitCode": proc.returncode,
"logsStderr": text(proc.stderr, 1000),
"valuesPrinted": False,
}
def validate_gateway_responses(api_key):
request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000))
payload = {
@@ -2427,7 +2489,7 @@ def summarize_temp_unschedulable_rules(rules):
"errorCode": rule.get("error_code"),
"durationMinutes": rule.get("duration_minutes"),
"keywordCount": len(rule.get("keywords") or []),
"keywords": (rule.get("keywords") or [])[:8],
"keywords": rule.get("keywords") or [],
"hasDescription": bool(rule.get("description")),
} for rule in rules]
@@ -2677,9 +2739,10 @@ def run_sync():
owner_concurrency = ensure_pool_owner_concurrency(token, api_key_result["userId"])
gateway = validate_gateway(api_key)
responses_smoke = validate_gateway_responses(api_key)
compact_evidence = recent_compact_gateway_evidence()
return {
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True,
"degraded": bool(responses_smoke.get("degraded")),
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")),
"mode": "sync",
"namespace": NAMESPACE,
"serviceDns": SERVICE_DNS,
@@ -2713,7 +2776,7 @@ def run_sync():
},
"ownerBalance": owner_balance,
"ownerConcurrency": owner_concurrency,
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke},
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayCompactRecent": compact_evidence},
}
def run_validate():
@@ -2733,9 +2796,10 @@ def run_validate():
temp_unschedulable_status = account_temp_unschedulable_status(token)
gateway = validate_gateway(api_key)
responses_smoke = validate_gateway_responses(api_key)
compact_evidence = recent_compact_gateway_evidence()
return {
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True,
"degraded": bool(responses_smoke.get("degraded")),
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")),
"mode": "validate",
"namespace": NAMESPACE,
"serviceDns": SERVICE_DNS,
@@ -2754,7 +2818,7 @@ def run_validate():
"loadFactor": load_factor_status,
"webSocketsV2": ws_v2_status,
"tempUnschedulable": temp_unschedulable_status,
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke},
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayCompactRecent": compact_evidence},
}
try: