mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: observe codex encrypted-context affinity failures

Browse Source
This commit is contained in:
Codex
2026-06-11 18:30:06 +00:00
parent f5cf701969
commit 83b7189a71
1 changed files with 62 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-sub2api-codex-sentinel.ts
+62 -14
View File
@@ -1348,10 +1348,14 @@ def gateway_failure_kind(message, payload, config):
))
if fallback_written or upstream_already_written or stream_failure:
return "gateway-stream-forward-failure"
final_compatibility_failure = any(token in error_text for token in (
session_affinity_failure = any(token in error_text for token in (
"encrypted content could not be decrypted",
"could not be verified",
"invalid_encrypted_content",
))
if session_affinity_failure:
return "gateway-session-affinity-failure"
final_compatibility_failure = any(token in error_text for token in (
"bad_response_status_code",
"model_not_found",
"no available channel for model",
@@ -1383,6 +1387,9 @@ def gateway_failure_kind(message, payload, config):
return "gateway-final-transient-failure"
return None
def gateway_failure_is_observe_only(failure_kind):
return failure_kind in {"gateway-session-affinity-failure"}
def gateway_failure_item(ts, pod_name, payload, failure_kind):
request_id = payload.get("request_id") or sha(json.dumps(payload, sort_keys=True, ensure_ascii=False))
try:
@@ -1506,6 +1513,30 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro
}
return action
def record_gateway_observation(account_name, failures, state, now):
latest = failures[-1]
account_state = state.setdefault("accounts", {}).setdefault(account_name, {})
account_state["lastGatewayAffinityFailureAt"] = iso(now)
account_state["lastGatewayAffinityFailure"] = {
"accountName": account_name,
"accountId": latest.get("accountId"),
"requestId": latest.get("requestId"),
"clientRequestId": latest.get("clientRequestId"),
"failureKind": latest.get("failureKind"),
"path": latest.get("path"),
"errorPreview": latest.get("errorPreview"),
"bodyBytes": latest.get("bodyBytes"),
"latencyMs": latest.get("latencyMs"),
"countInRun": len(failures),
"firstAt": failures[0].get("at"),
"lastAt": latest.get("at"),
"action": {
"taken": False,
"type": "observe-session-affinity-failure",
},
}
return {"taken": False, "type": "observe-session-affinity-failure"}
def run_gateway_failure_monitor(state, config, now, kube, admin, profiles):
cfg = config.get("gatewayFailureMonitor") if isinstance(config.get("gatewayFailureMonitor"), dict) else {}
if cfg.get("enabled") is not True:
@@ -1565,19 +1596,36 @@ def run_gateway_failure_monitor(state, config, now, kube, admin, profiles):
for account_name, failures in sorted(by_account.items()):
failures.sort(key=lambda item: item.get("at") or "")
profile = profile_by_name[account_name]
action = apply_gateway_failure(account_name, failures, state, config, now, admin, profile)
actions.append({
"accountName": account_name,
"accountId": failures[-1].get("accountId"),
"failureCount": len(failures),
"requestId": failures[-1].get("requestId"),
"failureKind": failures[-1].get("failureKind"),
"path": failures[-1].get("path"),
"errorPreview": failures[-1].get("errorPreview"),
"taken": action.get("taken"),
"type": action.get("type"),
"error": action.get("error"),
})
freeze_failures = [item for item in failures if not gateway_failure_is_observe_only(item.get("failureKind"))]
observed_failures = [item for item in failures if gateway_failure_is_observe_only(item.get("failureKind"))]
if freeze_failures:
action = apply_gateway_failure(account_name, freeze_failures, state, config, now, admin, profile)
actions.append({
"accountName": account_name,
"accountId": freeze_failures[-1].get("accountId"),
"failureCount": len(freeze_failures),
"requestId": freeze_failures[-1].get("requestId"),
"failureKind": freeze_failures[-1].get("failureKind"),
"path": freeze_failures[-1].get("path"),
"errorPreview": freeze_failures[-1].get("errorPreview"),
"taken": action.get("taken"),
"type": action.get("type"),
"error": action.get("error"),
})
if observed_failures:
action = record_gateway_observation(account_name, observed_failures, state, now)
actions.append({
"accountName": account_name,
"accountId": observed_failures[-1].get("accountId"),
"failureCount": len(observed_failures),
"requestId": observed_failures[-1].get("requestId"),
"failureKind": observed_failures[-1].get("failureKind"),
"path": observed_failures[-1].get("path"),
"errorPreview": observed_failures[-1].get("errorPreview"),
"taken": action.get("taken"),
"type": action.get("type"),
"error": action.get("error"),
})
monitor_state["lastRunAt"] = iso(now)
monitor_state["lastScannedPods"] = [((pod.get("metadata") or {}).get("name")) for pod in pods if isinstance(pod, dict)]
monitor_state["lastCandidateCount"] = len(candidates)