mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: protect primary sub2api codex account

Browse Source
This commit is contained in:
Codex
2026-06-13 05:05:31 +00:00
parent 90fae0c1c4
commit d99f57cf83
4 changed files with 289 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-sub2api-codex.ts
+125 -2
View File
@@ -105,6 +105,7 @@ interface CodexProfile {
openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
upstreamUserAgent: string | null;
trustUpstream: boolean;
sentinelProtect: CodexSentinelProtectPolicy;
priority: number;
capacity: number;
loadFactor: number;
@@ -156,12 +157,21 @@ interface CodexPoolProfileConfig {
openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
upstreamUserAgent: string | null;
trustUpstream: boolean;
sentinelProtect: CodexSentinelProtectPolicy;
priority: number;
capacity: number | null;
loadFactor: number | null;
tempUnschedulable: CodexTempUnschedulablePolicy;
}
export interface CodexSentinelProtectPolicy {
enabled: boolean;
consecutiveFailures: number;
initialRetryDelaySeconds: number;
maxRetryDelaySeconds: number;
backoffMultiplier: number;
}
interface CodexPoolPublicExposureConfig {
enabled: boolean;
proxyName: string;
@@ -714,10 +724,12 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi
upstreamUserAgent: profile.upstreamUserAgent,
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
trustUpstream: profile.trustUpstream,
sentinelProtect: profile.sentinelProtect,
}),
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
upstreamUserAgent: profile.upstreamUserAgent,
trustUpstream: profile.trustUpstream,
sentinelProtect: profile.sentinelProtect,
priority: profile.priority,
capacity: profile.capacity,
loadFactor: profile.loadFactor,
@@ -1098,6 +1110,7 @@ function collectCodexProfiles(): CodexProfile[] {
openaiResponsesWebSocketsV2Mode: entry.openaiResponsesWebSocketsV2Mode,
upstreamUserAgent: entry.upstreamUserAgent,
trustUpstream: entry.trustUpstream,
sentinelProtect: entry.sentinelProtect,
priority: entry.priority,
capacity: entry.capacity ?? pool.defaultAccountCapacity,
loadFactor: entry.loadFactor ?? pool.defaultAccountLoadFactor,
@@ -1171,6 +1184,7 @@ function discoverCodexProfileConfigs(
openaiResponsesWebSocketsV2Mode: null,
upstreamUserAgent: null,
trustUpstream: false,
sentinelProtect: defaultCodexSentinelProtectPolicy(),
priority: defaultPriority,
capacity: null,
loadFactor: null,
@@ -1340,6 +1354,7 @@ function readProfileConfig(
const openaiResponsesWebSocketsV2Mode = readOpenAIResponsesWebSocketsV2Mode(entry.openaiResponsesWebSocketsV2Mode, `profiles.entries[${index}].openaiResponsesWebSocketsV2Mode`);
const upstreamUserAgent = readUpstreamUserAgent(entry.upstreamUserAgent, `profiles.entries[${index}].upstreamUserAgent`);
const trustUpstream = readTrustUpstream(entry.trustUpstream, `profiles.entries[${index}].trustUpstream`);
const sentinelProtect = readSentinelProtectPolicy(entry.sentinelProtect, `profiles.entries[${index}].sentinelProtect`);
const priority = readAccountPriority(entry.priority, `profiles.entries[${index}].priority`, defaultPriority);
const capacity = entry.capacity === undefined || entry.capacity === null ? null : readAccountCapacity(entry.capacity, `profiles.entries[${index}].capacity`);
const loadFactor = entry.loadFactor === undefined || entry.loadFactor === null ? null : readAccountLoadFactor(entry.loadFactor, `profiles.entries[${index}].loadFactor`);
@@ -1354,6 +1369,7 @@ function readProfileConfig(
openaiResponsesWebSocketsV2Mode,
upstreamUserAgent,
trustUpstream,
sentinelProtect,
priority,
capacity,
loadFactor,
@@ -1396,6 +1412,52 @@ function readTrustUpstream(value: unknown, key: string): boolean {
return parsed;
}
export function defaultCodexSentinelProtectPolicy(): CodexSentinelProtectPolicy {
return {
enabled: false,
consecutiveFailures: 1,
initialRetryDelaySeconds: 2,
maxRetryDelaySeconds: 60,
backoffMultiplier: 2,
};
}
function readSentinelProtectPolicy(value: unknown, key: string): CodexSentinelProtectPolicy {
const fallback = defaultCodexSentinelProtectPolicy();
if (value === undefined || value === null) return fallback;
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback.enabled);
if (!enabled) return { ...fallback, enabled: false };
const required = ["consecutiveFailures", "initialRetryDelaySeconds", "maxRetryDelaySeconds", "backoffMultiplier"];
for (const field of required) {
if (value[field] === undefined || value[field] === null) {
throw new Error(`${codexPoolConfigPath}.${key}.${field} is required when enabled=true`);
}
}
const consecutiveFailures = readBoundedInteger(value.consecutiveFailures, `${key}.consecutiveFailures`, 1, 20);
const initialRetryDelaySeconds = readBoundedInteger(value.initialRetryDelaySeconds, `${key}.initialRetryDelaySeconds`, 1, 3600);
const maxRetryDelaySeconds = readBoundedInteger(value.maxRetryDelaySeconds, `${key}.maxRetryDelaySeconds`, 1, 3600);
const backoffMultiplier = readBoundedInteger(value.backoffMultiplier, `${key}.backoffMultiplier`, 1, 10);
if (maxRetryDelaySeconds < initialRetryDelaySeconds) {
throw new Error(`${codexPoolConfigPath}.${key}.maxRetryDelaySeconds must be >= initialRetryDelaySeconds`);
}
return {
enabled: true,
consecutiveFailures,
initialRetryDelaySeconds,
maxRetryDelaySeconds,
backoffMultiplier,
};
}
function readBoundedInteger(value: unknown, key: string, min: number, max: number): number {
const parsed = numberValue(value);
if (parsed === null || !Number.isInteger(parsed) || parsed < min || parsed > max) {
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from ${min} to ${max}`);
}
return parsed;
}
function readAccountPriority(value: unknown, key: string, fallback = defaultAccountPriority): number {
if (value === undefined || value === null) return fallback;
const priority = numberValue(value);
@@ -1723,6 +1785,7 @@ function redactProfile(profile: CodexProfile): Record<string, unknown> {
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
upstreamUserAgent: profile.upstreamUserAgent,
trustUpstream: profile.trustUpstream,
sentinelProtect: profile.sentinelProtect,
priority: profile.priority,
capacity: profile.capacity,
loadFactor: profile.loadFactor,
@@ -1745,6 +1808,8 @@ function compactProfile(profile: CodexProfile): Record<string, unknown> {
model: profile.model,
priority: profile.priority,
trustUpstream: profile.trustUpstream,
sentinelProtectEnabled: profile.sentinelProtect.enabled,
sentinelProtectConsecutiveFailures: profile.sentinelProtect.enabled ? profile.sentinelProtect.consecutiveFailures : undefined,
capacity: profile.capacity,
loadFactor: profile.loadFactor,
tempUnschedulableEnabled: profile.tempUnschedulable.enabled,
@@ -2061,6 +2126,7 @@ function compactSentinelErrorDetails(value: unknown): Record<string, unknown> |
function compactSentinelQuarantine(item: unknown): Record<string, unknown> {
if (!isRecord(item)) return {};
const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {};
return {
accountName: item.accountName,
until: item.until,
@@ -2068,6 +2134,12 @@ function compactSentinelQuarantine(item: unknown): Record<string, unknown> {
reason: item.reason,
failureKind: item.failureKind,
intervalMinutes: item.intervalMinutes,
sentinelProtect: Object.keys(protect).length > 0 ? {
enabled: protect.enabled,
decision: protect.decision,
failureCount: protect.failureCount,
threshold: protect.threshold,
} : undefined,
error: compactSentinelErrorDetails(item.errorDetails),
};
}
@@ -2076,6 +2148,7 @@ function compactSentinelRecentAccount(item: unknown): Record<string, unknown> {
if (!isRecord(item)) return {};
const action = isRecord(item.action) ? item.action : {};
const error = compactSentinelErrorDetails(item.errorDetails);
const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {};
return {
accountName: item.accountName,
lastProbeAt: item.lastProbeAt,
@@ -2086,6 +2159,13 @@ function compactSentinelRecentAccount(item: unknown): Record<string, unknown> {
httpStatus: item.httpStatus,
durationMs: item.durationMs,
markerMatched: item.markerMatched,
sentinelProtect: Object.keys(protect).length > 0 ? {
enabled: protect.enabled,
decision: protect.decision,
protected: protect.protected,
failureCount: protect.failureCount,
threshold: protect.threshold,
} : undefined,
failureKind: item.failureKind,
requestShape: item.requestShape,
action: Object.keys(action).length > 0 ? {
@@ -2238,6 +2318,7 @@ function compactSentinelProbeResult(parsed: Record<string, unknown> | null): Rec
"purpose",
"ok",
"markerMatched",
"sentinelProtect",
"httpStatus",
"durationMs",
"usage",
@@ -2307,12 +2388,14 @@ function renderSentinelReport(
lines.push("");
lines.push("ACCOUNTS");
lines.push(renderTable([
["ACCOUNT", "STATE", "Q", "T", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
["ACCOUNT", "STATE", "Q", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
...accounts.map((account) => [
stringValue(account.account) ?? "-",
stringValue(account.status) ?? "-",
account.quarantineActive === true ? "Y" : "-",
account.trustUpstream === true ? "Y" : account.trustUpstream === false ? "N" : "-",
account.sentinelProtectEnabled === true ? textValue(account.sentinelProtectThreshold) : "-",
account.sentinelProtectDecision ? `${textValue(account.sentinelProtectFailureCount)}/${textValue(account.sentinelProtectThreshold)}` : "-",
textValue(account.freezeIntervalMin),
textValue(account.successIntervalMin),
textValue(account.successMaxIntervalMin),
@@ -2346,7 +2429,7 @@ function renderSentinelReport(
]));
}
lines.push("");
lines.push("LEGEND Q=quarantined T=trusted upstream M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
lines.push("LEGEND Q=quarantined T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw");
return lines.join("\n");
}
@@ -2687,6 +2770,7 @@ function sentinelProfileSecrets(profiles: CodexProfile[]): CodexPoolSentinelProf
apiKey: profile.apiKey ?? "",
upstreamUserAgent: profile.upstreamUserAgent,
trustUpstream: profile.trustUpstream,
sentinelProtect: profile.sentinelProtect,
}));
}
@@ -3389,6 +3473,7 @@ export function codexPoolSentinelProbeConfigFingerprint(input: {
upstreamUserAgent: string | null;
openaiResponsesWebSocketsV2Mode: string | null;
trustUpstream: boolean;
sentinelProtect: CodexSentinelProtectPolicy;
}): string {
return fingerprint(JSON.stringify({
accountName: input.accountName,
@@ -3398,6 +3483,7 @@ export function codexPoolSentinelProbeConfigFingerprint(input: {
upstreamUserAgent: input.upstreamUserAgent,
openaiResponsesWebSocketsV2Mode: input.openaiResponsesWebSocketsV2Mode,
trustUpstream: input.trustUpstream,
sentinelProtect: input.sentinelProtect,
}));
}
@@ -3517,6 +3603,24 @@ def error_code(probe):
return openai_error.get("code") or openai_error.get("type") or ""
return details.get("kind") or ""
def sentinel_protect_summary(account_state, probe):
probe_protect = probe.get("sentinelProtect") if isinstance(probe.get("sentinelProtect"), dict) else {}
config_protect = account_state.get("sentinelProtectConfig") if isinstance(account_state.get("sentinelProtectConfig"), dict) else {}
source = probe_protect if probe_protect else config_protect
if not isinstance(source, dict) or source.get("enabled") is not True:
return {
"enabled": False,
"threshold": None,
"decision": None,
"failureCount": None,
}
return {
"enabled": True,
"threshold": source.get("threshold") or source.get("consecutiveFailures"),
"decision": probe_protect.get("decision"),
"failureCount": probe_protect.get("failureCount"),
}
def report():
cronjob, cron_error = kube_json(["-n", NAMESPACE, "get", "cronjob", CRONJOB_NAME])
state_cm, state_error = kube_json(["-n", NAMESPACE, "get", "configmap", STATE_NAME])
@@ -3544,6 +3648,7 @@ def report():
if gateway_failure and (not account_state.get("lastProbeAt") or str(account_state.get("lastGatewayFailureAt") or "") >= str(account_state.get("lastProbeAt") or "")):
last_action = action_type(gateway_failure)
ledger = account_ledger(account_state)
protect = sentinel_protect_summary(account_state, probe)
account_rows.append({
"account": name,
"status": account_state.get("lastStatus"),
@@ -3566,6 +3671,10 @@ def report():
"lastPurpose": probe.get("purpose"),
"lastHttp": probe.get("httpStatus"),
"lastMarker": probe.get("markerMatched"),
"sentinelProtectEnabled": protect.get("enabled"),
"sentinelProtectDecision": protect.get("decision"),
"sentinelProtectFailureCount": protect.get("failureCount"),
"sentinelProtectThreshold": protect.get("threshold"),
"lastFailureKind": last_failure_kind,
"lastErrorCode": error_code(probe),
"lastAction": last_action,
@@ -3987,6 +4096,8 @@ def sentinel_probe_change_reasons(current, profile):
runtime_ws_mode = empty_to_none(extra.get("openai_apikey_responses_websockets_v2_mode"))
expected_trust_upstream = profile.get("trustUpstream") is True
runtime_trust_upstream = extra.get("unidesk_trust_upstream") is True
expected_protect = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
runtime_protect = extra.get("unidesk_sentinel_protect") if isinstance(extra.get("unidesk_sentinel_protect"), dict) else {"enabled": False}
reasons = []
if empty_to_none(extra.get("unidesk_codex_profile")) != profile.get("profile"):
reasons.append("profile")
@@ -4000,6 +4111,8 @@ def sentinel_probe_change_reasons(current, profile):
reasons.append("responses-websockets-v2-mode")
if runtime_trust_upstream != expected_trust_upstream:
reasons.append("trust-upstream")
if runtime_protect != expected_protect:
reasons.append("sentinel-protect")
return reasons
def curl_api(method, path, bearer=None, payload=None):
@@ -4214,6 +4327,7 @@ def account_payload(profile, group_id):
"unidesk_codex_profile": profile["profile"],
"unidesk_managed": True,
"unidesk_trust_upstream": profile.get("trustUpstream") is True,
"unidesk_sentinel_protect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False},
}
ws_mode = profile.get("openaiResponsesWebSocketsV2Mode")
if ws_mode:
@@ -4264,6 +4378,7 @@ def planned_sentinel_account_results(profiles, existing_accounts):
"profileConfig": {
"accountName": profile["accountName"],
"trustUpstream": profile.get("trustUpstream") is True,
"sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False},
},
"sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"),
"sentinelProbeRequired": quality_gate_required,
@@ -4308,6 +4423,7 @@ def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_fr
"profileConfig": {
"accountName": profile["accountName"],
"trustUpstream": profile.get("trustUpstream") is True,
"sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False},
},
"accountId": data.get("id") if isinstance(data, dict) else None,
"action": action,
@@ -4467,6 +4583,7 @@ def sentinel_runtime_status():
"failureKind": quarantine.get("failureKind"),
"errorDetails": quarantine.get("errorDetails"),
"intervalMinutes": quarantine.get("intervalMinutes"),
"sentinelProtect": quarantine.get("sentinelProtect"),
})
last_probe = account_state.get("lastProbe")
if isinstance(last_probe, dict):
@@ -4480,6 +4597,7 @@ def sentinel_runtime_status():
"httpStatus": last_probe.get("httpStatus"),
"durationMs": last_probe.get("durationMs"),
"markerMatched": last_probe.get("markerMatched"),
"sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"),
"outputHash": last_probe.get("outputHash"),
"outputPreview": last_probe.get("outputPreview"),
"responseBodyHash": last_probe.get("responseBodyHash"),
@@ -4652,6 +4770,7 @@ def clamp_sentinel_success_cadence_for_config(state, profiles, now):
quarantine = account_state.get("quarantine")
if isinstance(quarantine, dict) and quarantine.get("active") is True:
account_state["trustUpstream"] = profile.get("trustUpstream") is True
account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile)
continue
try:
@@ -4661,6 +4780,7 @@ def clamp_sentinel_success_cadence_for_config(state, profiles, now):
next_epoch = parse_iso_epoch(account_state.get("nextProbeAfter"))
max_interval = profile_success_max_interval(profile)
account_state["trustUpstream"] = profile.get("trustUpstream") is True
account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
account_state["successMaxIntervalMinutes"] = max_interval
if interval <= max_interval and (next_epoch is None or next_epoch <= now_epoch + max_interval * 60):
continue
@@ -4751,6 +4871,7 @@ def ensure_sentinel_state_for_sync(account_results, pending_only=False):
account_state["successIntervalMinutes"] = 0
profile_config = item.get("profileConfig") if isinstance(item.get("profileConfig"), dict) else {}
account_state["trustUpstream"] = profile_config.get("trustUpstream") is True
account_state["sentinelProtectConfig"] = profile_config.get("sentinelProtect") if isinstance(profile_config.get("sentinelProtect"), dict) else {"enabled": False}
account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile_config)
account_state["lastStatus"] = "pending-sentinel-quality-gate"
account_state["qualityGate"] = {
@@ -4811,6 +4932,7 @@ def sentinel_state_summary():
"failureKind": quarantine.get("failureKind"),
"errorDetails": quarantine.get("errorDetails"),
"intervalMinutes": quarantine.get("intervalMinutes"),
"sentinelProtect": quarantine.get("sentinelProtect"),
})
last_probe = account_state.get("lastProbe")
if isinstance(last_probe, dict):
@@ -4824,6 +4946,7 @@ def sentinel_state_summary():
"httpStatus": last_probe.get("httpStatus"),
"durationMs": last_probe.get("durationMs"),
"markerMatched": last_probe.get("markerMatched"),
"sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"),
"usage": last_probe.get("usage"),
"responseBodyHash": last_probe.get("responseBodyHash"),
"responseBodyPreview": last_probe.get("responseBodyPreview"),