fix: cool sub2api upstream 400 responses failures

scripts/src/platform-infra-sub2api-codex.ts

@@ -676,6 +676,12 @@ export function defaultCodexTempUnschedulablePolicy(): CodexTempUnschedulablePol
         durationMinutes: 120,
         description: "Success-body account-state prompts require Sub2API 2xx body reclassification before they can cool accounts.",
       },
+      {
+        statusCode: 400,
+        keywords: ["invalid_encrypted_content", "encrypted content", "could not be verified", "could not be decrypted", "bad_response_status_code", "model_not_found", "no available channel for model", "unsupported", "not supported", "not support", "暂不支持", "可用模型"],
+        durationMinutes: 120,
+        description: "Stable upstream 400 model-routing or Responses encrypted-content compatibility failures should use another account.",
+      },
       {
         statusCode: 401,
         keywords: ["unauthorized", "invalid api key", "invalid_api_key", "authentication", "recovered upstream error"],
@@ -2412,6 +2418,76 @@ def recent_compact_gateway_evidence():
         "valuesPrinted": False,
     }
 
+def recent_responses_gateway_evidence():
+    proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", "--since=6h", "--tail=2500"])
+    stdout = proc.stdout.decode("utf-8", errors="replace")
+    failovers = []
+    forward_failures = []
+    final_errors = []
+    context_canceled = []
+    slow_final_errors = []
+    for line in stdout.splitlines():
+        if '"/responses"' not in line and '"/v1/responses"' not in line:
+            continue
+        json_start = line.find("{")
+        if json_start < 0:
+            continue
+        try:
+            item = json.loads(line[json_start:])
+        except Exception:
+            continue
+        path = item.get("path")
+        if path not in ("/responses", "/v1/responses"):
+            continue
+        entry = {
+            "requestId": item.get("request_id"),
+            "clientRequestId": item.get("client_request_id"),
+            "accountId": item.get("account_id"),
+            "statusCode": item.get("status_code"),
+            "upstreamStatus": item.get("upstream_status"),
+            "latencyMs": item.get("latency_ms"),
+            "path": path,
+        }
+        if "upstream_failover_switching" in line:
+            failovers.append({
+                **entry,
+                "switchCount": item.get("switch_count"),
+                "maxSwitches": item.get("max_switches"),
+            })
+        elif "openai.forward_failed" in line:
+            forward_failures.append({
+                **entry,
+                "errorPreview": text(str(item.get("error") or ""), 500),
+                "fallbackErrorResponseWritten": item.get("fallback_error_response_written"),
+                "upstreamErrorResponseAlreadyWritten": item.get("upstream_error_response_already_written"),
+            })
+        elif "http request completed" in line and isinstance(item.get("status_code"), int) and item.get("status_code") >= 400:
+            final_errors.append(entry)
+            latency_ms = item.get("latency_ms")
+            if isinstance(latency_ms, int) and latency_ms >= 30000:
+                slow_final_errors.append(entry)
+        if "context canceled" in line:
+            context_canceled.append(entry)
+    return {
+        "ok": True,
+        "degraded": len(forward_failures) > 0 or len(final_errors) > 0 or len(context_canceled) > 0,
+        "window": "6h",
+        "tailLines": 2500,
+        "failoverCount": len(failovers),
+        "forwardFailureCount": len(forward_failures),
+        "finalErrorCount": len(final_errors),
+        "slowFinalErrorCount": len(slow_final_errors),
+        "contextCanceledCount": len(context_canceled),
+        "recentFailovers": failovers[-8:],
+        "recentForwardFailures": forward_failures[-8:],
+        "recentFinalErrors": final_errors[-8:],
+        "recentSlowFinalErrors": slow_final_errors[-5:],
+        "recentContextCanceled": context_canceled[-5:],
+        "logsExitCode": proc.returncode,
+        "logsStderr": text(proc.stderr, 1000),
+        "valuesPrinted": False,
+    }
+
 def validate_gateway_responses(api_key):
     request_id = "unidesk-codex-pool-validate-" + str(int(time.time() * 1000))
     payload = {
@@ -3101,10 +3177,11 @@ def run_sync():
     gateway = validate_gateway(api_key)
     responses_smoke = validate_gateway_responses(api_key)
     compact_evidence = recent_compact_gateway_evidence()
+    responses_evidence = recent_responses_gateway_evidence()
     runtime_capabilities = validate_runtime_capabilities(token)
     return {
         "ok": gateway["ok"] is True and responses_smoke["ok"] is True and owner_concurrency["ok"] is True and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True,
-        "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
+        "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
         "mode": "sync",
         "namespace": NAMESPACE,
         "serviceDns": SERVICE_DNS,
@@ -3139,7 +3216,7 @@ def run_sync():
         "ownerBalance": owner_balance,
         "ownerConcurrency": owner_concurrency,
         "runtimeCapabilities": runtime_capabilities,
-        "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayCompactRecent": compact_evidence},
+        "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
     }
 
 def run_validate():
@@ -3160,10 +3237,11 @@ def run_validate():
     gateway = validate_gateway(api_key)
     responses_smoke = validate_gateway_responses(api_key)
     compact_evidence = recent_compact_gateway_evidence()
+    responses_evidence = recent_responses_gateway_evidence()
     runtime_capabilities = validate_runtime_capabilities(token)
     return {
         "ok": gateway["ok"] is True and responses_smoke["ok"] is True and (owner_concurrency is None or owner_concurrency["ok"] is True) and capacity_status["ok"] is True and load_factor_status["ok"] is True and ws_v2_status["ok"] is True and temp_unschedulable_status["ok"] is True,
-        "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
+        "degraded": bool(responses_smoke.get("degraded")) or bool(compact_evidence.get("degraded")) or bool(responses_evidence.get("degraded")) or runtime_capabilities.get("ok") is not True,
         "mode": "validate",
         "namespace": NAMESPACE,
         "serviceDns": SERVICE_DNS,
@@ -3183,7 +3261,7 @@ def run_validate():
         "webSocketsV2": ws_v2_status,
         "tempUnschedulable": temp_unschedulable_status,
         "runtimeCapabilities": runtime_capabilities,
-        "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayCompactRecent": compact_evidence},
+        "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence},
     }
 
 try: