fix: add compact timeout cooldown evidence
This commit is contained in:
Codex
@@ -694,7 +694,7 @@ export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePol
|
||||
},
|
||||
{
|
||||
statusCode: 502,
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "bad gateway", "upstream request failed", "websocket dial", "handshake response", "recovered upstream error"],
|
||||
keywords: ["capacity", "overloaded", "temporarily unavailable", "temporary", "upstream", "bad gateway", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "websocket dial", "handshake response", "recovered upstream error"],
|
||||
durationMinutes: 30,
|
||||
description: "Gateway upstream failures, including recovered upstream error wrappers, should cool down longer.",
|
||||
},
|
||||
@@ -718,7 +718,7 @@ export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePol
|
||||
},
|
||||
{
|
||||
statusCode: 524,
|
||||
keywords: ["timeout", "a timeout occurred", "cloudflare", "gateway timeout", "upstream", "context deadline exceeded"],
|
||||
keywords: ["timeout", "a timeout occurred", "cloudflare", "gateway timeout", "upstream", "upstream request failed", "unknown error", "context deadline exceeded", "context canceled", "recovered upstream error"],
|
||||
durationMinutes: 30,
|
||||
description: "Cloudflare 524 timeout responses should cool down the selected account so another account can handle the next request.",
|
||||
},
|
||||
@@ -2326,6 +2326,68 @@ def request_log_evidence(request_id):
|
||||
"logsStderr": text(proc.stderr, 1000),
|
||||
}
|
||||
|
||||
def recent_compact_gateway_evidence():
|
||||
proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"])
|
||||
stdout = proc.stdout.decode("utf-8", errors="replace")
|
||||
failures = []
|
||||
successes = []
|
||||
failovers = []
|
||||
final_errors = []
|
||||
context_canceled = []
|
||||
for line in stdout.splitlines():
|
||||
if "/responses/compact" not in line and "remote_compact" not in line:
|
||||
continue
|
||||
json_start = line.find("{")
|
||||
if json_start < 0:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line[json_start:])
|
||||
except Exception:
|
||||
continue
|
||||
path = item.get("path")
|
||||
entry = {
|
||||
"requestId": item.get("request_id"),
|
||||
"clientRequestId": item.get("client_request_id"),
|
||||
"accountId": item.get("account_id"),
|
||||
"statusCode": item.get("status_code"),
|
||||
"upstreamStatus": item.get("upstream_status"),
|
||||
"latencyMs": item.get("latency_ms"),
|
||||
"path": path,
|
||||
}
|
||||
if "codex.remote_compact.failed" in line:
|
||||
failures.append(entry)
|
||||
elif "codex.remote_compact.succeeded" in line:
|
||||
successes.append(entry)
|
||||
elif "upstream_failover_switching" in line and path == "/responses/compact":
|
||||
failovers.append({
|
||||
**entry,
|
||||
"switchCount": item.get("switch_count"),
|
||||
"maxSwitches": item.get("max_switches"),
|
||||
})
|
||||
elif "http request completed" in line and path == "/responses/compact" and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
|
||||
final_errors.append(entry)
|
||||
if "context canceled" in line and path == "/responses/compact":
|
||||
context_canceled.append(entry)
|
||||
return {
|
||||
"ok": True,
|
||||
"degraded": len(failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0,
|
||||
"window": "6h",
|
||||
"tailLines": 2500,
|
||||
"failureCount": len(failures),
|
||||
"successCount": len(successes),
|
||||
"failoverCount": len(failovers),
|
||||
"finalErrorCount": len(final_errors),
|
||||
"contextCanceledCount": len(context_canceled),
|
||||
"recentFailures": failures[-5:],
|
||||
"recentSuccesses": successes[-5:],
|
||||
"recentFailovers": failovers[-8:],
|
||||
"recentFinalErrors": final_errors[-5:],
|
||||
"recentContextCanceled": context_canceled[-5:],
|
||||
"logsExitCode": proc.returncode,
|
||||
"logsStderr": text(proc.stderr, 1000),
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
def validate_gateway_responses(api_key):
|
||||
request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000))
|
||||
payload = {
|
||||
@@ -2427,7 +2489,7 @@ def summarize_temp_unschedulable_rules(rules):
|
||||
"errorCode": rule.get("error_code"),
|
||||
"durationMinutes": rule.get("duration_minutes"),
|
||||
"keywordCount": len(rule.get("keywords") or []),
|
||||
"keywords": (rule.get("keywords") or [])[:8],
|
||||
"keywords": rule.get("keywords") or [],
|
||||
"hasDescription": bool(rule.get("description")),
|
||||
} for rule in rules]
|
||||
|
||||
@@ -2677,9 +2739,10 @@ def run_sync():
|
||||
owner_concurrency = ensure_pool_owner_concurrency(token, api_key_result["userId"])
|
||||
gateway = validate_gateway(api_key)
|
||||
responses_smoke = validate_gateway_responses(api_key)
|
||||
compact_evidence = recent_compact_gateway_evidence()
|
||||
return {
|
||||
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True,
|
||||
"degraded": bool(responses_smoke.get("degraded")),
|
||||
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")),
|
||||
"mode": "sync",
|
||||
"namespace": NAMESPACE,
|
||||
"serviceDns": SERVICE_DNS,
|
||||
@@ -2713,7 +2776,7 @@ def run_sync():
|
||||
},
|
||||
"ownerBalance": owner_balance,
|
||||
"ownerConcurrency": owner_concurrency,
|
||||
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke},
|
||||
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayCompactRecent": compact_evidence},
|
||||
}
|
||||
|
||||
def run_validate():
|
||||
@@ -2733,9 +2796,10 @@ def run_validate():
|
||||
temp_unschedulable_status = account_temp_unschedulable_status(token)
|
||||
gateway = validate_gateway(api_key)
|
||||
responses_smoke = validate_gateway_responses(api_key)
|
||||
compact_evidence = recent_compact_gateway_evidence()
|
||||
return {
|
||||
"ok": gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True,
|
||||
"degraded": bool(responses_smoke.get("degraded")),
|
||||
"degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")),
|
||||
"mode": "validate",
|
||||
"namespace": NAMESPACE,
|
||||
"serviceDns": SERVICE_DNS,
|
||||
@@ -2754,7 +2818,7 @@ def run_validate():
|
||||
"loadFactor": load_factor_status,
|
||||
"webSocketsV2": ws_v2_status,
|
||||
"tempUnschedulable": temp_unschedulable_status,
|
||||
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke},
|
||||
"validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayCompactRecent": compact_evidence},
|
||||
}
|
||||
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user