fix: protect primary sub2api codex account
This commit is contained in:
@@ -73,6 +73,15 @@ export interface CodexPoolSentinelProfileSecret {
|
||||
apiKey: string;
|
||||
upstreamUserAgent: string | null;
|
||||
trustUpstream: boolean;
|
||||
sentinelProtect: CodexPoolSentinelProtectPolicy;
|
||||
}
|
||||
|
||||
export interface CodexPoolSentinelProtectPolicy {
|
||||
enabled: boolean;
|
||||
consecutiveFailures: number;
|
||||
initialRetryDelaySeconds: number;
|
||||
maxRetryDelaySeconds: number;
|
||||
backoffMultiplier: number;
|
||||
}
|
||||
|
||||
export interface CodexPoolSentinelManifestOptions {
|
||||
@@ -1148,6 +1157,85 @@ def probe_account(profile, config, purpose):
|
||||
"requestShape": resp.get("requestShape"),
|
||||
}
|
||||
|
||||
def protect_policy(profile):
|
||||
policy = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {}
|
||||
return policy if policy.get("enabled") is True else None
|
||||
|
||||
def protect_failure_threshold(policy):
|
||||
try:
|
||||
return max(1, int(policy.get("consecutiveFailures") or 1))
|
||||
except Exception:
|
||||
return 1
|
||||
|
||||
def protect_retry_delay_seconds(policy, retry_index):
|
||||
try:
|
||||
initial = max(1, int(policy.get("initialRetryDelaySeconds") or 2))
|
||||
except Exception:
|
||||
initial = 2
|
||||
try:
|
||||
maximum = max(initial, int(policy.get("maxRetryDelaySeconds") or initial))
|
||||
except Exception:
|
||||
maximum = initial
|
||||
try:
|
||||
multiplier = max(1, int(policy.get("backoffMultiplier") or 2))
|
||||
except Exception:
|
||||
multiplier = 2
|
||||
if retry_index <= 0:
|
||||
return 0
|
||||
return min(maximum, initial * (multiplier ** (retry_index - 1)))
|
||||
|
||||
def probe_account_with_protection(profile, config, purpose):
|
||||
policy = protect_policy(profile)
|
||||
if policy is None:
|
||||
return probe_account(profile, config, purpose)
|
||||
threshold = protect_failure_threshold(policy)
|
||||
attempts = []
|
||||
last_result = None
|
||||
for index in range(threshold):
|
||||
delay = protect_retry_delay_seconds(policy, index)
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
attempt_purpose = purpose if index == 0 else purpose + "-protect-retry"
|
||||
result = probe_account(profile, config, attempt_purpose)
|
||||
attempt = {
|
||||
"attempt": index + 1,
|
||||
"delaySeconds": delay,
|
||||
"ok": result.get("ok"),
|
||||
"markerMatched": result.get("markerMatched"),
|
||||
"httpStatus": result.get("httpStatus"),
|
||||
"durationMs": result.get("durationMs"),
|
||||
"failureKind": result.get("failureKind"),
|
||||
"outputHash": result.get("outputHash"),
|
||||
"responseBodyHash": result.get("responseBodyHash"),
|
||||
"errorDetails": result.get("errorDetails"),
|
||||
}
|
||||
attempts.append(attempt)
|
||||
last_result = result
|
||||
if result.get("markerMatched") is True:
|
||||
result["sentinelProtect"] = {
|
||||
"enabled": True,
|
||||
"threshold": threshold,
|
||||
"attempts": attempts,
|
||||
"failureCount": index,
|
||||
"protected": index > 0,
|
||||
"decision": "pass",
|
||||
}
|
||||
if index > 0:
|
||||
result["purpose"] = purpose + "-protect-recovered"
|
||||
return result
|
||||
if last_result is None:
|
||||
last_result = probe_account(profile, config, purpose)
|
||||
last_result["sentinelProtect"] = {
|
||||
"enabled": True,
|
||||
"threshold": threshold,
|
||||
"attempts": attempts,
|
||||
"failureCount": len(attempts),
|
||||
"protected": False,
|
||||
"decision": "fail",
|
||||
}
|
||||
last_result["purpose"] = purpose + "-protect-exhausted"
|
||||
return last_result
|
||||
|
||||
def ledger_for(state, now):
|
||||
day = day_key(now)
|
||||
ledger = state.setdefault("ledger", {}).setdefault(day, {"inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "estimatedCostUsd": 0, "requestCount": 0})
|
||||
@@ -1306,6 +1394,7 @@ def apply_result(result, state, config, now, admin, profile):
|
||||
"ok": result.get("ok"),
|
||||
"purpose": result.get("purpose"),
|
||||
"trustUpstream": result.get("trustUpstream"),
|
||||
"sentinelProtect": result.get("sentinelProtect"),
|
||||
"successMaxIntervalMinutes": success_max_interval(profile, config),
|
||||
"httpStatus": result.get("httpStatus"),
|
||||
"durationMs": result.get("durationMs"),
|
||||
@@ -1479,6 +1568,68 @@ def next_gateway_freeze_interval(account_state, config):
|
||||
def apply_gateway_failure(account_name, failures, state, config, now, admin, profile):
|
||||
latest = failures[-1]
|
||||
account_state = state.setdefault("accounts", {}).setdefault(account_name, {})
|
||||
policy = protect_policy(profile)
|
||||
protected_probe = None
|
||||
if policy is not None:
|
||||
protected_probe = probe_account_with_protection(profile, config, "gateway-failure-confirm")
|
||||
protected_probe["sourceGatewayFailure"] = {
|
||||
"requestId": latest.get("requestId"),
|
||||
"clientRequestId": latest.get("clientRequestId"),
|
||||
"failureKind": latest.get("failureKind"),
|
||||
"path": latest.get("path"),
|
||||
"countInRun": len(failures),
|
||||
}
|
||||
account_state["lastProbeAt"] = iso(now)
|
||||
account_state["lastProbe"] = {
|
||||
"ok": protected_probe.get("ok"),
|
||||
"purpose": protected_probe.get("purpose"),
|
||||
"trustUpstream": protected_probe.get("trustUpstream"),
|
||||
"sentinelProtect": protected_probe.get("sentinelProtect"),
|
||||
"successMaxIntervalMinutes": success_max_interval(profile, config),
|
||||
"httpStatus": protected_probe.get("httpStatus"),
|
||||
"durationMs": protected_probe.get("durationMs"),
|
||||
"markerMatched": protected_probe.get("markerMatched"),
|
||||
"transportOk": protected_probe.get("transportOk"),
|
||||
"outputHash": protected_probe.get("outputHash"),
|
||||
"outputPreview": protected_probe.get("outputPreview"),
|
||||
"responseBodyHash": protected_probe.get("responseBodyHash"),
|
||||
"responseBodyPreview": protected_probe.get("responseBodyPreview"),
|
||||
"error": protected_probe.get("error"),
|
||||
"errorDetails": protected_probe.get("errorDetails"),
|
||||
"usage": protected_probe.get("usage"),
|
||||
"failureKind": protected_probe.get("failureKind"),
|
||||
"sdk": protected_probe.get("sdk"),
|
||||
"requestShape": protected_probe.get("requestShape"),
|
||||
"action": {"taken": False, "type": "protect-confirm-pass" if protected_probe.get("markerMatched") is True else "protect-confirm-fail"},
|
||||
"sourceGatewayFailure": protected_probe.get("sourceGatewayFailure"),
|
||||
}
|
||||
add_usage(state, account_state, now, protected_probe.get("usage") or {})
|
||||
account_state["sentinelProtect"] = protected_probe.get("sentinelProtect")
|
||||
account_state["trustUpstream"] = profile.get("trustUpstream") is True
|
||||
account_state["successMaxIntervalMinutes"] = success_max_interval(profile, config)
|
||||
if protected_probe.get("markerMatched") is True:
|
||||
interval = next_success_interval(account_state, config, profile)
|
||||
account_state["successStreak"] = int(account_state.get("successStreak") or 0) + 1
|
||||
account_state["successIntervalMinutes"] = interval
|
||||
account_state["nextProbeAfter"] = iso(add_minutes(now, interval, int(config["cadence"]["jitterPercent"])))
|
||||
account_state["lastOkAt"] = iso(now)
|
||||
account_state["lastStatus"] = "gateway-failure-protect-confirmed-ok"
|
||||
account_state["lastGatewayFailureAt"] = iso(now)
|
||||
account_state["lastGatewayFailure"] = {
|
||||
"accountName": account_name,
|
||||
"accountId": latest.get("accountId"),
|
||||
"requestId": latest.get("requestId"),
|
||||
"clientRequestId": latest.get("clientRequestId"),
|
||||
"failureKind": latest.get("failureKind"),
|
||||
"path": latest.get("path"),
|
||||
"errorPreview": latest.get("errorPreview"),
|
||||
"countInRun": len(failures),
|
||||
"firstAt": failures[0].get("at"),
|
||||
"lastAt": latest.get("at"),
|
||||
"action": {"taken": False, "type": "protect-confirm-pass"},
|
||||
"sentinelProtect": protected_probe.get("sentinelProtect"),
|
||||
}
|
||||
return {"taken": False, "type": "protect-confirm-pass", "sentinelProtect": protected_probe.get("sentinelProtect")}
|
||||
interval = next_gateway_freeze_interval(account_state, config)
|
||||
until = add_minutes(now, interval, int(config["freeze"]["jitterPercent"]))
|
||||
actions_enabled = bool(config["actions"]["enabled"])
|
||||
@@ -1511,6 +1662,7 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro
|
||||
"countInRun": len(failures),
|
||||
},
|
||||
"lastBadAt": iso(now),
|
||||
"sentinelProtect": protected_probe.get("sentinelProtect") if isinstance(protected_probe, dict) else None,
|
||||
}
|
||||
account_state["nextProbeAfter"] = iso(until)
|
||||
account_state["successStreak"] = 0
|
||||
@@ -1538,6 +1690,7 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro
|
||||
"intervalMinutes": interval,
|
||||
"freezeUntil": iso(until),
|
||||
"action": action,
|
||||
"sentinelProtect": protected_probe.get("sentinelProtect") if isinstance(protected_probe, dict) else None,
|
||||
}
|
||||
return action
|
||||
|
||||
@@ -1721,7 +1874,7 @@ def main():
|
||||
actions = []
|
||||
if (config["monitor"]["enabled"] or forced_names) and due:
|
||||
with ThreadPoolExecutor(max_workers=max(1, len(due))) as executor:
|
||||
futures = {executor.submit(probe_account, item["profile"], config, item["purpose"]): item["profile"] for item in due}
|
||||
futures = {executor.submit(probe_account_with_protection, item["profile"], config, item["purpose"]): item["profile"] for item in due}
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
@@ -1760,6 +1913,7 @@ def main():
|
||||
"accountName": item.get("accountName"),
|
||||
"purpose": item.get("purpose"),
|
||||
"trustUpstream": item.get("trustUpstream"),
|
||||
"sentinelProtect": item.get("sentinelProtect"),
|
||||
"ok": item.get("ok"),
|
||||
"markerMatched": item.get("markerMatched"),
|
||||
"httpStatus": item.get("httpStatus"),
|
||||
|
||||
@@ -105,6 +105,7 @@ interface CodexProfile {
|
||||
openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
|
||||
upstreamUserAgent: string | null;
|
||||
trustUpstream: boolean;
|
||||
sentinelProtect: CodexSentinelProtectPolicy;
|
||||
priority: number;
|
||||
capacity: number;
|
||||
loadFactor: number;
|
||||
@@ -156,12 +157,21 @@ interface CodexPoolProfileConfig {
|
||||
openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null;
|
||||
upstreamUserAgent: string | null;
|
||||
trustUpstream: boolean;
|
||||
sentinelProtect: CodexSentinelProtectPolicy;
|
||||
priority: number;
|
||||
capacity: number | null;
|
||||
loadFactor: number | null;
|
||||
tempUnschedulable: CodexTempUnschedulablePolicy;
|
||||
}
|
||||
|
||||
export interface CodexSentinelProtectPolicy {
|
||||
enabled: boolean;
|
||||
consecutiveFailures: number;
|
||||
initialRetryDelaySeconds: number;
|
||||
maxRetryDelaySeconds: number;
|
||||
backoffMultiplier: number;
|
||||
}
|
||||
|
||||
interface CodexPoolPublicExposureConfig {
|
||||
enabled: boolean;
|
||||
proxyName: string;
|
||||
@@ -714,10 +724,12 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi
|
||||
upstreamUserAgent: profile.upstreamUserAgent,
|
||||
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
|
||||
trustUpstream: profile.trustUpstream,
|
||||
sentinelProtect: profile.sentinelProtect,
|
||||
}),
|
||||
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
|
||||
upstreamUserAgent: profile.upstreamUserAgent,
|
||||
trustUpstream: profile.trustUpstream,
|
||||
sentinelProtect: profile.sentinelProtect,
|
||||
priority: profile.priority,
|
||||
capacity: profile.capacity,
|
||||
loadFactor: profile.loadFactor,
|
||||
@@ -1098,6 +1110,7 @@ function collectCodexProfiles(): CodexProfile[] {
|
||||
openaiResponsesWebSocketsV2Mode: entry.openaiResponsesWebSocketsV2Mode,
|
||||
upstreamUserAgent: entry.upstreamUserAgent,
|
||||
trustUpstream: entry.trustUpstream,
|
||||
sentinelProtect: entry.sentinelProtect,
|
||||
priority: entry.priority,
|
||||
capacity: entry.capacity ?? pool.defaultAccountCapacity,
|
||||
loadFactor: entry.loadFactor ?? pool.defaultAccountLoadFactor,
|
||||
@@ -1171,6 +1184,7 @@ function discoverCodexProfileConfigs(
|
||||
openaiResponsesWebSocketsV2Mode: null,
|
||||
upstreamUserAgent: null,
|
||||
trustUpstream: false,
|
||||
sentinelProtect: defaultCodexSentinelProtectPolicy(),
|
||||
priority: defaultPriority,
|
||||
capacity: null,
|
||||
loadFactor: null,
|
||||
@@ -1340,6 +1354,7 @@ function readProfileConfig(
|
||||
const openaiResponsesWebSocketsV2Mode = readOpenAIResponsesWebSocketsV2Mode(entry.openaiResponsesWebSocketsV2Mode, `profiles.entries[${index}].openaiResponsesWebSocketsV2Mode`);
|
||||
const upstreamUserAgent = readUpstreamUserAgent(entry.upstreamUserAgent, `profiles.entries[${index}].upstreamUserAgent`);
|
||||
const trustUpstream = readTrustUpstream(entry.trustUpstream, `profiles.entries[${index}].trustUpstream`);
|
||||
const sentinelProtect = readSentinelProtectPolicy(entry.sentinelProtect, `profiles.entries[${index}].sentinelProtect`);
|
||||
const priority = readAccountPriority(entry.priority, `profiles.entries[${index}].priority`, defaultPriority);
|
||||
const capacity = entry.capacity === undefined || entry.capacity === null ? null : readAccountCapacity(entry.capacity, `profiles.entries[${index}].capacity`);
|
||||
const loadFactor = entry.loadFactor === undefined || entry.loadFactor === null ? null : readAccountLoadFactor(entry.loadFactor, `profiles.entries[${index}].loadFactor`);
|
||||
@@ -1354,6 +1369,7 @@ function readProfileConfig(
|
||||
openaiResponsesWebSocketsV2Mode,
|
||||
upstreamUserAgent,
|
||||
trustUpstream,
|
||||
sentinelProtect,
|
||||
priority,
|
||||
capacity,
|
||||
loadFactor,
|
||||
@@ -1396,6 +1412,52 @@ function readTrustUpstream(value: unknown, key: string): boolean {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function defaultCodexSentinelProtectPolicy(): CodexSentinelProtectPolicy {
|
||||
return {
|
||||
enabled: false,
|
||||
consecutiveFailures: 1,
|
||||
initialRetryDelaySeconds: 2,
|
||||
maxRetryDelaySeconds: 60,
|
||||
backoffMultiplier: 2,
|
||||
};
|
||||
}
|
||||
|
||||
function readSentinelProtectPolicy(value: unknown, key: string): CodexSentinelProtectPolicy {
|
||||
const fallback = defaultCodexSentinelProtectPolicy();
|
||||
if (value === undefined || value === null) return fallback;
|
||||
if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`);
|
||||
const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback.enabled);
|
||||
if (!enabled) return { ...fallback, enabled: false };
|
||||
const required = ["consecutiveFailures", "initialRetryDelaySeconds", "maxRetryDelaySeconds", "backoffMultiplier"];
|
||||
for (const field of required) {
|
||||
if (value[field] === undefined || value[field] === null) {
|
||||
throw new Error(`${codexPoolConfigPath}.${key}.${field} is required when enabled=true`);
|
||||
}
|
||||
}
|
||||
const consecutiveFailures = readBoundedInteger(value.consecutiveFailures, `${key}.consecutiveFailures`, 1, 20);
|
||||
const initialRetryDelaySeconds = readBoundedInteger(value.initialRetryDelaySeconds, `${key}.initialRetryDelaySeconds`, 1, 3600);
|
||||
const maxRetryDelaySeconds = readBoundedInteger(value.maxRetryDelaySeconds, `${key}.maxRetryDelaySeconds`, 1, 3600);
|
||||
const backoffMultiplier = readBoundedInteger(value.backoffMultiplier, `${key}.backoffMultiplier`, 1, 10);
|
||||
if (maxRetryDelaySeconds < initialRetryDelaySeconds) {
|
||||
throw new Error(`${codexPoolConfigPath}.${key}.maxRetryDelaySeconds must be >= initialRetryDelaySeconds`);
|
||||
}
|
||||
return {
|
||||
enabled: true,
|
||||
consecutiveFailures,
|
||||
initialRetryDelaySeconds,
|
||||
maxRetryDelaySeconds,
|
||||
backoffMultiplier,
|
||||
};
|
||||
}
|
||||
|
||||
function readBoundedInteger(value: unknown, key: string, min: number, max: number): number {
|
||||
const parsed = numberValue(value);
|
||||
if (parsed === null || !Number.isInteger(parsed) || parsed < min || parsed > max) {
|
||||
throw new Error(`${codexPoolConfigPath}.${key} must be an integer from ${min} to ${max}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readAccountPriority(value: unknown, key: string, fallback = defaultAccountPriority): number {
|
||||
if (value === undefined || value === null) return fallback;
|
||||
const priority = numberValue(value);
|
||||
@@ -1723,6 +1785,7 @@ function redactProfile(profile: CodexProfile): Record<string, unknown> {
|
||||
openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode,
|
||||
upstreamUserAgent: profile.upstreamUserAgent,
|
||||
trustUpstream: profile.trustUpstream,
|
||||
sentinelProtect: profile.sentinelProtect,
|
||||
priority: profile.priority,
|
||||
capacity: profile.capacity,
|
||||
loadFactor: profile.loadFactor,
|
||||
@@ -1745,6 +1808,8 @@ function compactProfile(profile: CodexProfile): Record<string, unknown> {
|
||||
model: profile.model,
|
||||
priority: profile.priority,
|
||||
trustUpstream: profile.trustUpstream,
|
||||
sentinelProtectEnabled: profile.sentinelProtect.enabled,
|
||||
sentinelProtectConsecutiveFailures: profile.sentinelProtect.enabled ? profile.sentinelProtect.consecutiveFailures : undefined,
|
||||
capacity: profile.capacity,
|
||||
loadFactor: profile.loadFactor,
|
||||
tempUnschedulableEnabled: profile.tempUnschedulable.enabled,
|
||||
@@ -2061,6 +2126,7 @@ function compactSentinelErrorDetails(value: unknown): Record<string, unknown> |
|
||||
|
||||
function compactSentinelQuarantine(item: unknown): Record<string, unknown> {
|
||||
if (!isRecord(item)) return {};
|
||||
const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {};
|
||||
return {
|
||||
accountName: item.accountName,
|
||||
until: item.until,
|
||||
@@ -2068,6 +2134,12 @@ function compactSentinelQuarantine(item: unknown): Record<string, unknown> {
|
||||
reason: item.reason,
|
||||
failureKind: item.failureKind,
|
||||
intervalMinutes: item.intervalMinutes,
|
||||
sentinelProtect: Object.keys(protect).length > 0 ? {
|
||||
enabled: protect.enabled,
|
||||
decision: protect.decision,
|
||||
failureCount: protect.failureCount,
|
||||
threshold: protect.threshold,
|
||||
} : undefined,
|
||||
error: compactSentinelErrorDetails(item.errorDetails),
|
||||
};
|
||||
}
|
||||
@@ -2076,6 +2148,7 @@ function compactSentinelRecentAccount(item: unknown): Record<string, unknown> {
|
||||
if (!isRecord(item)) return {};
|
||||
const action = isRecord(item.action) ? item.action : {};
|
||||
const error = compactSentinelErrorDetails(item.errorDetails);
|
||||
const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {};
|
||||
return {
|
||||
accountName: item.accountName,
|
||||
lastProbeAt: item.lastProbeAt,
|
||||
@@ -2086,6 +2159,13 @@ function compactSentinelRecentAccount(item: unknown): Record<string, unknown> {
|
||||
httpStatus: item.httpStatus,
|
||||
durationMs: item.durationMs,
|
||||
markerMatched: item.markerMatched,
|
||||
sentinelProtect: Object.keys(protect).length > 0 ? {
|
||||
enabled: protect.enabled,
|
||||
decision: protect.decision,
|
||||
protected: protect.protected,
|
||||
failureCount: protect.failureCount,
|
||||
threshold: protect.threshold,
|
||||
} : undefined,
|
||||
failureKind: item.failureKind,
|
||||
requestShape: item.requestShape,
|
||||
action: Object.keys(action).length > 0 ? {
|
||||
@@ -2238,6 +2318,7 @@ function compactSentinelProbeResult(parsed: Record<string, unknown> | null): Rec
|
||||
"purpose",
|
||||
"ok",
|
||||
"markerMatched",
|
||||
"sentinelProtect",
|
||||
"httpStatus",
|
||||
"durationMs",
|
||||
"usage",
|
||||
@@ -2307,12 +2388,14 @@ function renderSentinelReport(
|
||||
lines.push("");
|
||||
lines.push("ACCOUNTS");
|
||||
lines.push(renderTable([
|
||||
["ACCOUNT", "STATE", "Q", "T", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
|
||||
["ACCOUNT", "STATE", "Q", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
|
||||
...accounts.map((account) => [
|
||||
stringValue(account.account) ?? "-",
|
||||
stringValue(account.status) ?? "-",
|
||||
account.quarantineActive === true ? "Y" : "-",
|
||||
account.trustUpstream === true ? "Y" : account.trustUpstream === false ? "N" : "-",
|
||||
account.sentinelProtectEnabled === true ? textValue(account.sentinelProtectThreshold) : "-",
|
||||
account.sentinelProtectDecision ? `${textValue(account.sentinelProtectFailureCount)}/${textValue(account.sentinelProtectThreshold)}` : "-",
|
||||
textValue(account.freezeIntervalMin),
|
||||
textValue(account.successIntervalMin),
|
||||
textValue(account.successMaxIntervalMin),
|
||||
@@ -2346,7 +2429,7 @@ function renderSentinelReport(
|
||||
]));
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("LEGEND Q=quarantined T=trusted upstream M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
|
||||
lines.push("LEGEND Q=quarantined T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
|
||||
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw");
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -2687,6 +2770,7 @@ function sentinelProfileSecrets(profiles: CodexProfile[]): CodexPoolSentinelProf
|
||||
apiKey: profile.apiKey ?? "",
|
||||
upstreamUserAgent: profile.upstreamUserAgent,
|
||||
trustUpstream: profile.trustUpstream,
|
||||
sentinelProtect: profile.sentinelProtect,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -3389,6 +3473,7 @@ export function codexPoolSentinelProbeConfigFingerprint(input: {
|
||||
upstreamUserAgent: string | null;
|
||||
openaiResponsesWebSocketsV2Mode: string | null;
|
||||
trustUpstream: boolean;
|
||||
sentinelProtect: CodexSentinelProtectPolicy;
|
||||
}): string {
|
||||
return fingerprint(JSON.stringify({
|
||||
accountName: input.accountName,
|
||||
@@ -3398,6 +3483,7 @@ export function codexPoolSentinelProbeConfigFingerprint(input: {
|
||||
upstreamUserAgent: input.upstreamUserAgent,
|
||||
openaiResponsesWebSocketsV2Mode: input.openaiResponsesWebSocketsV2Mode,
|
||||
trustUpstream: input.trustUpstream,
|
||||
sentinelProtect: input.sentinelProtect,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -3517,6 +3603,24 @@ def error_code(probe):
|
||||
return openai_error.get("code") or openai_error.get("type") or ""
|
||||
return details.get("kind") or ""
|
||||
|
||||
def sentinel_protect_summary(account_state, probe):
|
||||
probe_protect = probe.get("sentinelProtect") if isinstance(probe.get("sentinelProtect"), dict) else {}
|
||||
config_protect = account_state.get("sentinelProtectConfig") if isinstance(account_state.get("sentinelProtectConfig"), dict) else {}
|
||||
source = probe_protect if probe_protect else config_protect
|
||||
if not isinstance(source, dict) or source.get("enabled") is not True:
|
||||
return {
|
||||
"enabled": False,
|
||||
"threshold": None,
|
||||
"decision": None,
|
||||
"failureCount": None,
|
||||
}
|
||||
return {
|
||||
"enabled": True,
|
||||
"threshold": source.get("threshold") or source.get("consecutiveFailures"),
|
||||
"decision": probe_protect.get("decision"),
|
||||
"failureCount": probe_protect.get("failureCount"),
|
||||
}
|
||||
|
||||
def report():
|
||||
cronjob, cron_error = kube_json(["-n", NAMESPACE, "get", "cronjob", CRONJOB_NAME])
|
||||
state_cm, state_error = kube_json(["-n", NAMESPACE, "get", "configmap", STATE_NAME])
|
||||
@@ -3544,6 +3648,7 @@ def report():
|
||||
if gateway_failure and (not account_state.get("lastProbeAt") or str(account_state.get("lastGatewayFailureAt") or "") >= str(account_state.get("lastProbeAt") or "")):
|
||||
last_action = action_type(gateway_failure)
|
||||
ledger = account_ledger(account_state)
|
||||
protect = sentinel_protect_summary(account_state, probe)
|
||||
account_rows.append({
|
||||
"account": name,
|
||||
"status": account_state.get("lastStatus"),
|
||||
@@ -3566,6 +3671,10 @@ def report():
|
||||
"lastPurpose": probe.get("purpose"),
|
||||
"lastHttp": probe.get("httpStatus"),
|
||||
"lastMarker": probe.get("markerMatched"),
|
||||
"sentinelProtectEnabled": protect.get("enabled"),
|
||||
"sentinelProtectDecision": protect.get("decision"),
|
||||
"sentinelProtectFailureCount": protect.get("failureCount"),
|
||||
"sentinelProtectThreshold": protect.get("threshold"),
|
||||
"lastFailureKind": last_failure_kind,
|
||||
"lastErrorCode": error_code(probe),
|
||||
"lastAction": last_action,
|
||||
@@ -3987,6 +4096,8 @@ def sentinel_probe_change_reasons(current, profile):
|
||||
runtime_ws_mode = empty_to_none(extra.get("openai_apikey_responses_websockets_v2_mode"))
|
||||
expected_trust_upstream = profile.get("trustUpstream") is True
|
||||
runtime_trust_upstream = extra.get("unidesk_trust_upstream") is True
|
||||
expected_protect = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
|
||||
runtime_protect = extra.get("unidesk_sentinel_protect") if isinstance(extra.get("unidesk_sentinel_protect"), dict) else {"enabled": False}
|
||||
reasons = []
|
||||
if empty_to_none(extra.get("unidesk_codex_profile")) != profile.get("profile"):
|
||||
reasons.append("profile")
|
||||
@@ -4000,6 +4111,8 @@ def sentinel_probe_change_reasons(current, profile):
|
||||
reasons.append("responses-websockets-v2-mode")
|
||||
if runtime_trust_upstream != expected_trust_upstream:
|
||||
reasons.append("trust-upstream")
|
||||
if runtime_protect != expected_protect:
|
||||
reasons.append("sentinel-protect")
|
||||
return reasons
|
||||
|
||||
def curl_api(method, path, bearer=None, payload=None):
|
||||
@@ -4214,6 +4327,7 @@ def account_payload(profile, group_id):
|
||||
"unidesk_codex_profile": profile["profile"],
|
||||
"unidesk_managed": True,
|
||||
"unidesk_trust_upstream": profile.get("trustUpstream") is True,
|
||||
"unidesk_sentinel_protect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False},
|
||||
}
|
||||
ws_mode = profile.get("openaiResponsesWebSocketsV2Mode")
|
||||
if ws_mode:
|
||||
@@ -4264,6 +4378,7 @@ def planned_sentinel_account_results(profiles, existing_accounts):
|
||||
"profileConfig": {
|
||||
"accountName": profile["accountName"],
|
||||
"trustUpstream": profile.get("trustUpstream") is True,
|
||||
"sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False},
|
||||
},
|
||||
"sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"),
|
||||
"sentinelProbeRequired": quality_gate_required,
|
||||
@@ -4308,6 +4423,7 @@ def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_fr
|
||||
"profileConfig": {
|
||||
"accountName": profile["accountName"],
|
||||
"trustUpstream": profile.get("trustUpstream") is True,
|
||||
"sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False},
|
||||
},
|
||||
"accountId": data.get("id") if isinstance(data, dict) else None,
|
||||
"action": action,
|
||||
@@ -4467,6 +4583,7 @@ def sentinel_runtime_status():
|
||||
"failureKind": quarantine.get("failureKind"),
|
||||
"errorDetails": quarantine.get("errorDetails"),
|
||||
"intervalMinutes": quarantine.get("intervalMinutes"),
|
||||
"sentinelProtect": quarantine.get("sentinelProtect"),
|
||||
})
|
||||
last_probe = account_state.get("lastProbe")
|
||||
if isinstance(last_probe, dict):
|
||||
@@ -4480,6 +4597,7 @@ def sentinel_runtime_status():
|
||||
"httpStatus": last_probe.get("httpStatus"),
|
||||
"durationMs": last_probe.get("durationMs"),
|
||||
"markerMatched": last_probe.get("markerMatched"),
|
||||
"sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"),
|
||||
"outputHash": last_probe.get("outputHash"),
|
||||
"outputPreview": last_probe.get("outputPreview"),
|
||||
"responseBodyHash": last_probe.get("responseBodyHash"),
|
||||
@@ -4652,6 +4770,7 @@ def clamp_sentinel_success_cadence_for_config(state, profiles, now):
|
||||
quarantine = account_state.get("quarantine")
|
||||
if isinstance(quarantine, dict) and quarantine.get("active") is True:
|
||||
account_state["trustUpstream"] = profile.get("trustUpstream") is True
|
||||
account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
|
||||
account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile)
|
||||
continue
|
||||
try:
|
||||
@@ -4661,6 +4780,7 @@ def clamp_sentinel_success_cadence_for_config(state, profiles, now):
|
||||
next_epoch = parse_iso_epoch(account_state.get("nextProbeAfter"))
|
||||
max_interval = profile_success_max_interval(profile)
|
||||
account_state["trustUpstream"] = profile.get("trustUpstream") is True
|
||||
account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}
|
||||
account_state["successMaxIntervalMinutes"] = max_interval
|
||||
if interval <= max_interval and (next_epoch is None or next_epoch <= now_epoch + max_interval * 60):
|
||||
continue
|
||||
@@ -4751,6 +4871,7 @@ def ensure_sentinel_state_for_sync(account_results, pending_only=False):
|
||||
account_state["successIntervalMinutes"] = 0
|
||||
profile_config = item.get("profileConfig") if isinstance(item.get("profileConfig"), dict) else {}
|
||||
account_state["trustUpstream"] = profile_config.get("trustUpstream") is True
|
||||
account_state["sentinelProtectConfig"] = profile_config.get("sentinelProtect") if isinstance(profile_config.get("sentinelProtect"), dict) else {"enabled": False}
|
||||
account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile_config)
|
||||
account_state["lastStatus"] = "pending-sentinel-quality-gate"
|
||||
account_state["qualityGate"] = {
|
||||
@@ -4811,6 +4932,7 @@ def sentinel_state_summary():
|
||||
"failureKind": quarantine.get("failureKind"),
|
||||
"errorDetails": quarantine.get("errorDetails"),
|
||||
"intervalMinutes": quarantine.get("intervalMinutes"),
|
||||
"sentinelProtect": quarantine.get("sentinelProtect"),
|
||||
})
|
||||
last_probe = account_state.get("lastProbe")
|
||||
if isinstance(last_probe, dict):
|
||||
@@ -4824,6 +4946,7 @@ def sentinel_state_summary():
|
||||
"httpStatus": last_probe.get("httpStatus"),
|
||||
"durationMs": last_probe.get("durationMs"),
|
||||
"markerMatched": last_probe.get("markerMatched"),
|
||||
"sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"),
|
||||
"usage": last_probe.get("usage"),
|
||||
"responseBodyHash": last_probe.get("responseBodyHash"),
|
||||
"responseBodyPreview": last_probe.get("responseBodyPreview"),
|
||||
|
||||
Reference in New Issue
Block a user