diff --git a/scripts/src/platform-infra-sub2api-codex-sentinel.ts b/scripts/src/platform-infra-sub2api-codex-sentinel.ts index c91b7d25..ef1835fd 100644 --- a/scripts/src/platform-infra-sub2api-codex-sentinel.ts +++ b/scripts/src/platform-infra-sub2api-codex-sentinel.ts @@ -1348,10 +1348,14 @@ def gateway_failure_kind(message, payload, config): )) if fallback_written or upstream_already_written or stream_failure: return "gateway-stream-forward-failure" - final_compatibility_failure = any(token in error_text for token in ( + session_affinity_failure = any(token in error_text for token in ( "encrypted content could not be decrypted", "could not be verified", "invalid_encrypted_content", + )) + if session_affinity_failure: + return "gateway-session-affinity-failure" + final_compatibility_failure = any(token in error_text for token in ( "bad_response_status_code", "model_not_found", "no available channel for model", @@ -1383,6 +1387,9 @@ def gateway_failure_kind(message, payload, config): return "gateway-final-transient-failure" return None +def gateway_failure_is_observe_only(failure_kind): + return failure_kind in {"gateway-session-affinity-failure"} + def gateway_failure_item(ts, pod_name, payload, failure_kind): request_id = payload.get("request_id") or sha(json.dumps(payload, sort_keys=True, ensure_ascii=False)) try: @@ -1506,6 +1513,30 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro } return action +def record_gateway_observation(account_name, failures, state, now): + latest = failures[-1] + account_state = state.setdefault("accounts", {}).setdefault(account_name, {}) + account_state["lastGatewayAffinityFailureAt"] = iso(now) + account_state["lastGatewayAffinityFailure"] = { + "accountName": account_name, + "accountId": latest.get("accountId"), + "requestId": latest.get("requestId"), + "clientRequestId": latest.get("clientRequestId"), + "failureKind": latest.get("failureKind"), + "path": latest.get("path"), + "errorPreview": latest.get("errorPreview"), + "bodyBytes": latest.get("bodyBytes"), + "latencyMs": latest.get("latencyMs"), + "countInRun": len(failures), + "firstAt": failures[0].get("at"), + "lastAt": latest.get("at"), + "action": { + "taken": False, + "type": "observe-session-affinity-failure", + }, + } + return {"taken": False, "type": "observe-session-affinity-failure"} + def run_gateway_failure_monitor(state, config, now, kube, admin, profiles): cfg = config.get("gatewayFailureMonitor") if isinstance(config.get("gatewayFailureMonitor"), dict) else {} if cfg.get("enabled") is not True: @@ -1565,19 +1596,36 @@ def run_gateway_failure_monitor(state, config, now, kube, admin, profiles): for account_name, failures in sorted(by_account.items()): failures.sort(key=lambda item: item.get("at") or "") profile = profile_by_name[account_name] - action = apply_gateway_failure(account_name, failures, state, config, now, admin, profile) - actions.append({ - "accountName": account_name, - "accountId": failures[-1].get("accountId"), - "failureCount": len(failures), - "requestId": failures[-1].get("requestId"), - "failureKind": failures[-1].get("failureKind"), - "path": failures[-1].get("path"), - "errorPreview": failures[-1].get("errorPreview"), - "taken": action.get("taken"), - "type": action.get("type"), - "error": action.get("error"), - }) + freeze_failures = [item for item in failures if not gateway_failure_is_observe_only(item.get("failureKind"))] + observed_failures = [item for item in failures if gateway_failure_is_observe_only(item.get("failureKind"))] + if freeze_failures: + action = apply_gateway_failure(account_name, freeze_failures, state, config, now, admin, profile) + actions.append({ + "accountName": account_name, + "accountId": freeze_failures[-1].get("accountId"), + "failureCount": len(freeze_failures), + "requestId": freeze_failures[-1].get("requestId"), + "failureKind": freeze_failures[-1].get("failureKind"), + "path": freeze_failures[-1].get("path"), + "errorPreview": freeze_failures[-1].get("errorPreview"), + "taken": action.get("taken"), + "type": action.get("type"), + "error": action.get("error"), + }) + if observed_failures: + action = record_gateway_observation(account_name, observed_failures, state, now) + actions.append({ + "accountName": account_name, + "accountId": observed_failures[-1].get("accountId"), + "failureCount": len(observed_failures), + "requestId": observed_failures[-1].get("requestId"), + "failureKind": observed_failures[-1].get("failureKind"), + "path": observed_failures[-1].get("path"), + "errorPreview": observed_failures[-1].get("errorPreview"), + "taken": action.get("taken"), + "type": action.get("type"), + "error": action.get("error"), + }) monitor_state["lastRunAt"] = iso(now) monitor_state["lastScannedPods"] = [((pod.get("metadata") or {}).get("name")) for pod in pods if isinstance(pod, dict)] monitor_state["lastCandidateCount"] = len(candidates)