fix: protect primary sub2api codex account

2026-06-13 05:05:31 +00:00
parent 90fae0c1c4
commit d99f57cf83
4 changed files with 289 additions and 5 deletions
@@ -73,6 +73,15 @@ export interface CodexPoolSentinelProfileSecret {
  apiKey: string;
  upstreamUserAgent: string | null;
  trustUpstream: boolean;
+  sentinelProtect: CodexPoolSentinelProtectPolicy;
+}
+
+export interface CodexPoolSentinelProtectPolicy {
+  enabled: boolean;
+  consecutiveFailures: number;
+  initialRetryDelaySeconds: number;
+  maxRetryDelaySeconds: number;
+  backoffMultiplier: number;
 }

 export interface CodexPoolSentinelManifestOptions {
@@ -1148,6 +1157,85 @@ def probe_account(profile, config, purpose):
        "requestShape": resp.get("requestShape"),
    }

+def protect_policy(profile):
+    policy = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {}
+    return policy if policy.get("enabled") is True else None
+
+def protect_failure_threshold(policy):
+    try:
+        return max(1, int(policy.get("consecutiveFailures") or 1))
+    except Exception:
+        return 1
+
+def protect_retry_delay_seconds(policy, retry_index):
+    try:
+        initial = max(1, int(policy.get("initialRetryDelaySeconds") or 2))
+    except Exception:
+        initial = 2
+    try:
+        maximum = max(initial, int(policy.get("maxRetryDelaySeconds") or initial))
+    except Exception:
+        maximum = initial
+    try:
+        multiplier = max(1, int(policy.get("backoffMultiplier") or 2))
+    except Exception:
+        multiplier = 2
+    if retry_index <= 0:
+        return 0
+    return min(maximum, initial * (multiplier ** (retry_index - 1)))
+
+def probe_account_with_protection(profile, config, purpose):
+    policy = protect_policy(profile)
+    if policy is None:
+        return probe_account(profile, config, purpose)
+    threshold = protect_failure_threshold(policy)
+    attempts = []
+    last_result = None
+    for index in range(threshold):
+        delay = protect_retry_delay_seconds(policy, index)
+        if delay > 0:
+            time.sleep(delay)
+        attempt_purpose = purpose if index == 0 else purpose + "-protect-retry"
+        result = probe_account(profile, config, attempt_purpose)
+        attempt = {
+            "attempt": index + 1,
+            "delaySeconds": delay,
+            "ok": result.get("ok"),
+            "markerMatched": result.get("markerMatched"),
+            "httpStatus": result.get("httpStatus"),
+            "durationMs": result.get("durationMs"),
+            "failureKind": result.get("failureKind"),
+            "outputHash": result.get("outputHash"),
+            "responseBodyHash": result.get("responseBodyHash"),
+            "errorDetails": result.get("errorDetails"),
+        }
+        attempts.append(attempt)
+        last_result = result
+        if result.get("markerMatched") is True:
+            result["sentinelProtect"] = {
+                "enabled": True,
+                "threshold": threshold,
+                "attempts": attempts,
+                "failureCount": index,
+                "protected": index > 0,
+                "decision": "pass",
+            }
+            if index > 0:
+                result["purpose"] = purpose + "-protect-recovered"
+            return result
+    if last_result is None:
+        last_result = probe_account(profile, config, purpose)
+    last_result["sentinelProtect"] = {
+        "enabled": True,
+        "threshold": threshold,
+        "attempts": attempts,
+        "failureCount": len(attempts),
+        "protected": False,
+        "decision": "fail",
+    }
+    last_result["purpose"] = purpose + "-protect-exhausted"
+    return last_result
+
 def ledger_for(state, now):
    day = day_key(now)
    ledger = state.setdefault("ledger", {}).setdefault(day, {"inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "estimatedCostUsd": 0, "requestCount": 0})
@@ -1306,6 +1394,7 @@ def apply_result(result, state, config, now, admin, profile):
        "ok": result.get("ok"),
        "purpose": result.get("purpose"),
        "trustUpstream": result.get("trustUpstream"),
+        "sentinelProtect": result.get("sentinelProtect"),
        "successMaxIntervalMinutes": success_max_interval(profile, config),
        "httpStatus": result.get("httpStatus"),
        "durationMs": result.get("durationMs"),
@@ -1479,6 +1568,68 @@ def next_gateway_freeze_interval(account_state, config):
 def apply_gateway_failure(account_name, failures, state, config, now, admin, profile):
    latest = failures[-1]
    account_state = state.setdefault("accounts", {}).setdefault(account_name, {})
+    policy = protect_policy(profile)
+    protected_probe = None
+    if policy is not None:
+        protected_probe = probe_account_with_protection(profile, config, "gateway-failure-confirm")
+        protected_probe["sourceGatewayFailure"] = {
+            "requestId": latest.get("requestId"),
+            "clientRequestId": latest.get("clientRequestId"),
+            "failureKind": latest.get("failureKind"),
+            "path": latest.get("path"),
+            "countInRun": len(failures),
+        }
+        account_state["lastProbeAt"] = iso(now)
+        account_state["lastProbe"] = {
+            "ok": protected_probe.get("ok"),
+            "purpose": protected_probe.get("purpose"),
+            "trustUpstream": protected_probe.get("trustUpstream"),
+            "sentinelProtect": protected_probe.get("sentinelProtect"),
+            "successMaxIntervalMinutes": success_max_interval(profile, config),
+            "httpStatus": protected_probe.get("httpStatus"),
+            "durationMs": protected_probe.get("durationMs"),
+            "markerMatched": protected_probe.get("markerMatched"),
+            "transportOk": protected_probe.get("transportOk"),
+            "outputHash": protected_probe.get("outputHash"),
+            "outputPreview": protected_probe.get("outputPreview"),
+            "responseBodyHash": protected_probe.get("responseBodyHash"),
+            "responseBodyPreview": protected_probe.get("responseBodyPreview"),
+            "error": protected_probe.get("error"),
+            "errorDetails": protected_probe.get("errorDetails"),
+            "usage": protected_probe.get("usage"),
+            "failureKind": protected_probe.get("failureKind"),
+            "sdk": protected_probe.get("sdk"),
+            "requestShape": protected_probe.get("requestShape"),
+            "action": {"taken": False, "type": "protect-confirm-pass" if protected_probe.get("markerMatched") is True else "protect-confirm-fail"},
+            "sourceGatewayFailure": protected_probe.get("sourceGatewayFailure"),
+        }
+        add_usage(state, account_state, now, protected_probe.get("usage") or {})
+        account_state["sentinelProtect"] = protected_probe.get("sentinelProtect")
+        account_state["trustUpstream"] = profile.get("trustUpstream") is True
+        account_state["successMaxIntervalMinutes"] = success_max_interval(profile, config)
+        if protected_probe.get("markerMatched") is True:
+            interval = next_success_interval(account_state, config, profile)
+            account_state["successStreak"] = int(account_state.get("successStreak") or 0) + 1
+            account_state["successIntervalMinutes"] = interval
+            account_state["nextProbeAfter"] = iso(add_minutes(now, interval, int(config["cadence"]["jitterPercent"])))
+            account_state["lastOkAt"] = iso(now)
+            account_state["lastStatus"] = "gateway-failure-protect-confirmed-ok"
+            account_state["lastGatewayFailureAt"] = iso(now)
+            account_state["lastGatewayFailure"] = {
+                "accountName": account_name,
+                "accountId": latest.get("accountId"),
+                "requestId": latest.get("requestId"),
+                "clientRequestId": latest.get("clientRequestId"),
+                "failureKind": latest.get("failureKind"),
+                "path": latest.get("path"),
+                "errorPreview": latest.get("errorPreview"),
+                "countInRun": len(failures),
+                "firstAt": failures[0].get("at"),
+                "lastAt": latest.get("at"),
+                "action": {"taken": False, "type": "protect-confirm-pass"},
+                "sentinelProtect": protected_probe.get("sentinelProtect"),
+            }
+            return {"taken": False, "type": "protect-confirm-pass", "sentinelProtect": protected_probe.get("sentinelProtect")}
    interval = next_gateway_freeze_interval(account_state, config)
    until = add_minutes(now, interval, int(config["freeze"]["jitterPercent"]))
    actions_enabled = bool(config["actions"]["enabled"])
@@ -1511,6 +1662,7 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro
            "countInRun": len(failures),
        },
        "lastBadAt": iso(now),
+        "sentinelProtect": protected_probe.get("sentinelProtect") if isinstance(protected_probe, dict) else None,
    }
    account_state["nextProbeAfter"] = iso(until)
    account_state["successStreak"] = 0
@@ -1538,6 +1690,7 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro
        "intervalMinutes": interval,
        "freezeUntil": iso(until),
        "action": action,
+        "sentinelProtect": protected_probe.get("sentinelProtect") if isinstance(protected_probe, dict) else None,
    }
    return action

@@ -1721,7 +1874,7 @@ def main():
    actions = []
    if (config["monitor"]["enabled"] or forced_names) and due:
        with ThreadPoolExecutor(max_workers=max(1, len(due))) as executor:
-            futures = {executor.submit(probe_account, item["profile"], config, item["purpose"]): item["profile"] for item in due}
+            futures = {executor.submit(probe_account_with_protection, item["profile"], config, item["purpose"]): item["profile"] for item in due}
            for future in as_completed(futures):
                result = future.result()
                results.append(result)
@@ -1760,6 +1913,7 @@ def main():
            "accountName": item.get("accountName"),
            "purpose": item.get("purpose"),
            "trustUpstream": item.get("trustUpstream"),
+            "sentinelProtect": item.get("sentinelProtect"),
            "ok": item.get("ok"),
            "markerMatched": item.get("markerMatched"),
            "httpStatus": item.get("httpStatus"),