mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: freeze codex accounts on gateway stream failures

Browse Source
This commit is contained in:
Codex
2026-06-11 17:06:37 +00:00
parent 170b538fe6
commit 35c18784a2
3 changed files with 340 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-sub2api-codex.ts
+20 -6
View File
@@ -1869,6 +1869,7 @@ function compactSentinelProbeResult(parsed: Record<string, unknown> | null): Rec
markerMismatchCount: summary.markerMismatchCount,
transportFailureCount: summary.transportFailureCount,
actionsTaken: summary.actionsTaken,
gatewayFailureMonitor: summary.gatewayFailureMonitor,
selection: summary.selection,
},
results: recordArray(probe.results).map((item) => pickSummaryFields(item, [
@@ -1955,7 +1956,7 @@ function renderSentinelReport(
textValue(account.successIntervalMin),
textValue(account.successMaxIntervalMin),
textValue(account.probeCount),
shortIso(account.lastProbeAt),
shortIso(account.lastEventAt ?? account.lastProbeAt),
textValue(account.lastHttp),
account.lastMarker === true ? "Y" : account.lastMarker === false ? "N" : "-",
shorten(stringValue(account.lastFailureKind) ?? "-", 20),
@@ -1968,7 +1969,7 @@ function renderSentinelReport(
lines.push("");
lines.push(`RUNS last=${Math.min(context.events, runs.length)}`);
lines.push(renderTable([
["AT", "SEL", "DUE", "OK", "BAD", "TF", "ACT", "REASSERT"],
["AT", "SEL", "DUE", "OK", "BAD", "TF", "ACT", "GF", "GACT", "REASSERT"],
...runs.slice(-context.events).map((run) => [
shortIso(run.at),
textValue(run.selected),
@@ -1977,12 +1978,14 @@ function renderSentinelReport(
textValue(run.mismatch),
textValue(run.transportFailures),
textValue(run.actionsTaken),
textValue(run.gatewayFailures),
textValue(run.gatewayActions),
textValue(run.reasserts),
]),
]));
}
lines.push("");
lines.push("LEGEND Q=quarantined T=trusted upstream M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last probe to next probe minutes TF=transport failures");
lines.push("LEGEND Q=quarantined T=trusted upstream M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw");
return lines.join("\n");
}
@@ -2973,6 +2976,12 @@ def report():
continue
probe = account_state.get("lastProbe") if isinstance(account_state.get("lastProbe"), dict) else {}
quarantine = account_state.get("quarantine") if isinstance(account_state.get("quarantine"), dict) else {}
gateway_failure = account_state.get("lastGatewayFailure") if isinstance(account_state.get("lastGatewayFailure"), dict) else {}
last_event_at = account_state.get("lastGatewayFailureAt") or account_state.get("lastProbeAt")
last_failure_kind = quarantine.get("failureKind") if quarantine.get("active") is True and quarantine.get("failureKind") else probe.get("failureKind")
last_action = action_type(probe)
if gateway_failure and (not account_state.get("lastProbeAt") or str(account_state.get("lastGatewayFailureAt") or "") >= str(account_state.get("lastProbeAt") or "")):
last_action = action_type(gateway_failure)
ledger = account_ledger(account_state)
account_rows.append({
"account": name,
@@ -2991,14 +3000,16 @@ def report():
"totalTokens": ledger.get("totalTokens", 0),
"estimatedCostUsd": round(float(ledger.get("estimatedCostUsd", 0)), 6),
"lastProbeAt": account_state.get("lastProbeAt"),
"lastEventAt": last_event_at,
"lastGatewayFailureAt": account_state.get("lastGatewayFailureAt"),
"lastPurpose": probe.get("purpose"),
"lastHttp": probe.get("httpStatus"),
"lastMarker": probe.get("markerMatched"),
"lastFailureKind": probe.get("failureKind"),
"lastFailureKind": last_failure_kind,
"lastErrorCode": error_code(probe),
"lastAction": action_type(probe),
"lastAction": last_action,
"nextProbeAfter": account_state.get("nextProbeAfter"),
"observedLastToNextMin": minutes_between(account_state.get("lastProbeAt"), account_state.get("nextProbeAfter")),
"observedLastToNextMin": minutes_between(last_event_at, account_state.get("nextProbeAfter")),
"requestShape": probe.get("requestShape"),
})
run_rows = []
@@ -3006,6 +3017,7 @@ def report():
if not isinstance(item, dict):
continue
selection = item.get("selection") if isinstance(item.get("selection"), dict) else {}
gateway = item.get("gatewayFailureMonitor") if isinstance(item.get("gatewayFailureMonitor"), dict) else {}
run_rows.append({
"at": item.get("at"),
"selected": item.get("selected"),
@@ -3014,6 +3026,8 @@ def report():
"mismatch": item.get("mismatchCount") if item.get("mismatchCount") is not None else item.get("markerMismatchCount"),
"transportFailures": item.get("transportFailureCount"),
"actionsTaken": item.get("actionsTaken"),
"gatewayFailures": gateway.get("newFailures"),
"gatewayActions": gateway.get("actionsTaken"),
"reasserts": len(item.get("reconcile") or []),
})
quarantined = [item for item in account_rows if item.get("quarantineActive") is True]