diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index 60b91678..3dfd5cad 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -105,6 +105,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 - `profiles.entries[].capacity`: 可选 per-account concurrency override；不写则使用 `pool.defaultAccountCapacity`。具体数值只以 `config/platform-infra/sub2api-codex-pool.yaml` 为准，skill 和长期参考只描述规则，不重复写当前值。 - `profiles.entries[].loadFactor`: 可选 per-account Sub2API `load_factor` override；不写则使用 `pool.defaultAccountLoadFactor`。具体数值只以 `config/platform-infra/sub2api-codex-pool.yaml` 为准，修改后必须 `codex-pool sync --confirm` 和 `codex-pool validate`。 - `profiles.entries[].trustUpstream`: 可选账号级哨兵信任标记；默认 `false`。可信账号使用 `sentinel.cadence.trustedSuccessMaxIntervalMinutes` 作为连续成功后的最大探测退避，不可信账号使用 `sentinel.cadence.untrustedSuccessMaxIntervalMinutes`。它只影响哨兵探测频率和状态可见性，不改变 Sub2API account priority/capacity/loadFactor。 +- `profiles.entries[].sentinelProtect`: 可选账号级哨兵保护策略；默认关闭，只给需要避免偶发 marker 波动误屏蔽的高价值账号启用。启用时必须显式声明 `consecutiveFailures`、`initialRetryDelaySeconds`、`maxRetryDelaySeconds` 和 `backoffMultiplier`；marker probe 或 gateway failure 触发冻结前会先按该策略做连续 marker 确认，只有全部失败才进入冻结状态机。它只影响哨兵冻结判定和 `sentinel-report` 可见性，不改变 Sub2API account priority/capacity/loadFactor。 - 除非用户明确要求修改配置，不要仅凭推断改账号 membership、priority、capacity、loadFactor、WebSocket mode 或其他调度策略；先保留 YAML，完成 provenance/runtime evidence 溯源，并把结论写回相关 issue 或 runbook 后再提出变更。 - Sub2API 是 UniDesk 可读源码和可观测运行面的受控组件；排查 Sub2API 调度、failover、错误传播、临时不可调度或 account selection 时，默认先读当前 Sub2API 源码实现，再用真实 request id、Sub2API 日志和原入口流量验证。不要用 mock upstream、临时 probe account 或测试桩作为默认结论来源；这类探针最多是显式 debug 辅助，不能替代源码链路和真实运行证据。 - `profiles.entries[].tempUnschedulable`: 可选 per-account Sub2API 内置临时不可调度覆盖；只用于明确偏离 pool 默认规则，不用它给某个账号特殊优先级或临时绕过通用 failover。 @@ -127,7 +128,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 `trace --request-id ` 是只读 request 追溯报表，不触发 probe、不修改账号。默认输出请求开始/最终状态、failover、`account_select_failed`、窗口内 `account_temp_unschedulable`、admin schedulable 写入计数和当前账号快照；`reason=failover-attempted-no-candidate` 表示 Sub2API 已进入自动切号，但排除当前失败账号后没有可用候选。需要机器处理时使用 `--raw`，需要原始匹配行时加 `--show-lines`。 -`sentinel-report` 是只读低噪声报表，不触发 probe、不修改账号。默认输出类似 `ps` 的文本表，展示每个账号的探测次数、最近 marker/HTTP/动作、冻结 TTL、成功退避、下一次 probe 和最近 run 事件；需要机器处理时使用 `sentinel-report --raw`。 +`sentinel-report` 是只读低噪声报表，不触发 probe、不修改账号。默认输出类似 `ps` 的文本表，展示每个账号的探测次数、最近 marker/HTTP/动作、冻结 TTL、成功退避、下一次 probe 和最近 run 事件；`PROT` 展示账号级保护阈值，`P_FAIL` 展示最近一次保护确认中的失败次数/阈值；需要机器处理时使用 `sentinel-report --raw`。 `sync --confirm` 和 `validate` 可能超过单次 SSH/runtime 短连接窗口。必须继续使用 `bun scripts/cli.ts platform-infra sub2api codex-pool ...`，由 CLI 在 G14 远端提交作业并短轮询状态；不要改用裸 `trans G14:k3s script` 等一个长连接等待完整结果。若看到 `UNIDESK_SSH_RUNTIME_TIMEOUT`，先按 `docs/reference/platform-infra.md` 的规则处理为控制面可见性问题，修 CLI/job/poll 或重跑受控命令，不要手工 patch Sub2API credentials 或源码。 @@ -141,7 +142,7 @@ Codex 启动时反复出现 WebSocket reconnect、HTTPS fallback、`websocket cl 1. 在 master `~/.codex/` 准备带后缀的上游 profile 文件，例如 `config.toml.` 和 `auth.json.`；禁止覆盖默认 `config.toml` / `auth.json`。 2. 在 `config/platform-infra/sub2api-codex-pool.yaml` 添加 `profiles.entries` 项，指定 `profile`、`accountName`、`configFile`、`authFile`。 -3. 如需要，给该项加 `priority`、`capacity`、`loadFactor`、`trustUpstream`、`openaiResponsesWebSocketsV2Mode` 或 `upstreamUserAgent`；capacity/loadFactor/信任退避的具体数值只写在 YAML。只有显式恢复 Sub2API 内置临时不可调度时才添加 per-account `tempUnschedulable`。 +3. 如需要，给该项加 `priority`、`capacity`、`loadFactor`、`trustUpstream`、`sentinelProtect`、`openaiResponsesWebSocketsV2Mode` 或 `upstreamUserAgent`；capacity/loadFactor/信任退避/保护阈值的具体数值只写在 YAML。只有显式恢复 Sub2API 内置临时不可调度时才添加 per-account `tempUnschedulable`。 4. 如果新增账号会提高声明 capacity 总和，默认让省略的 `pool.minOwnerConcurrency` 继续按 capacity 总和自动解析；只有 YAML 已经显式写了该 override 时，才同步提高到不低于总 capacity，或删除 override 回到自动解析。 5. 跑 `codex-pool plan`，确认 profile 可读、`base_url` 和 API key 来源有效，且 stdout 未泄露完整 key。 6. 跑 `codex-pool sync --confirm`。 diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 1d21c5e5..e5c6d4e5 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -90,6 +90,12 @@ profiles: configFile: config.toml.only authFile: auth.json.only trustUpstream: true + sentinelProtect: + enabled: true + consecutiveFailures: 3 + initialRetryDelaySeconds: 2 + maxRetryDelaySeconds: 60 + backoffMultiplier: 2 loadFactor: 1 priority: 110 - profile: zakuzaku diff --git a/scripts/src/platform-infra-sub2api-codex-sentinel.ts b/scripts/src/platform-infra-sub2api-codex-sentinel.ts index fcb08174..50657ada 100644 --- a/scripts/src/platform-infra-sub2api-codex-sentinel.ts +++ b/scripts/src/platform-infra-sub2api-codex-sentinel.ts @@ -73,6 +73,15 @@ export interface CodexPoolSentinelProfileSecret { apiKey: string; upstreamUserAgent: string | null; trustUpstream: boolean; + sentinelProtect: CodexPoolSentinelProtectPolicy; +} + +export interface CodexPoolSentinelProtectPolicy { + enabled: boolean; + consecutiveFailures: number; + initialRetryDelaySeconds: number; + maxRetryDelaySeconds: number; + backoffMultiplier: number; } export interface CodexPoolSentinelManifestOptions { @@ -1148,6 +1157,85 @@ def probe_account(profile, config, purpose): "requestShape": resp.get("requestShape"), } +def protect_policy(profile): + policy = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {} + return policy if policy.get("enabled") is True else None + +def protect_failure_threshold(policy): + try: + return max(1, int(policy.get("consecutiveFailures") or 1)) + except Exception: + return 1 + +def protect_retry_delay_seconds(policy, retry_index): + try: + initial = max(1, int(policy.get("initialRetryDelaySeconds") or 2)) + except Exception: + initial = 2 + try: + maximum = max(initial, int(policy.get("maxRetryDelaySeconds") or initial)) + except Exception: + maximum = initial + try: + multiplier = max(1, int(policy.get("backoffMultiplier") or 2)) + except Exception: + multiplier = 2 + if retry_index <= 0: + return 0 + return min(maximum, initial * (multiplier ** (retry_index - 1))) + +def probe_account_with_protection(profile, config, purpose): + policy = protect_policy(profile) + if policy is None: + return probe_account(profile, config, purpose) + threshold = protect_failure_threshold(policy) + attempts = [] + last_result = None + for index in range(threshold): + delay = protect_retry_delay_seconds(policy, index) + if delay > 0: + time.sleep(delay) + attempt_purpose = purpose if index == 0 else purpose + "-protect-retry" + result = probe_account(profile, config, attempt_purpose) + attempt = { + "attempt": index + 1, + "delaySeconds": delay, + "ok": result.get("ok"), + "markerMatched": result.get("markerMatched"), + "httpStatus": result.get("httpStatus"), + "durationMs": result.get("durationMs"), + "failureKind": result.get("failureKind"), + "outputHash": result.get("outputHash"), + "responseBodyHash": result.get("responseBodyHash"), + "errorDetails": result.get("errorDetails"), + } + attempts.append(attempt) + last_result = result + if result.get("markerMatched") is True: + result["sentinelProtect"] = { + "enabled": True, + "threshold": threshold, + "attempts": attempts, + "failureCount": index, + "protected": index > 0, + "decision": "pass", + } + if index > 0: + result["purpose"] = purpose + "-protect-recovered" + return result + if last_result is None: + last_result = probe_account(profile, config, purpose) + last_result["sentinelProtect"] = { + "enabled": True, + "threshold": threshold, + "attempts": attempts, + "failureCount": len(attempts), + "protected": False, + "decision": "fail", + } + last_result["purpose"] = purpose + "-protect-exhausted" + return last_result + def ledger_for(state, now): day = day_key(now) ledger = state.setdefault("ledger", {}).setdefault(day, {"inputTokens": 0, "outputTokens": 0, "totalTokens": 0, "estimatedCostUsd": 0, "requestCount": 0}) @@ -1306,6 +1394,7 @@ def apply_result(result, state, config, now, admin, profile): "ok": result.get("ok"), "purpose": result.get("purpose"), "trustUpstream": result.get("trustUpstream"), + "sentinelProtect": result.get("sentinelProtect"), "successMaxIntervalMinutes": success_max_interval(profile, config), "httpStatus": result.get("httpStatus"), "durationMs": result.get("durationMs"), @@ -1479,6 +1568,68 @@ def next_gateway_freeze_interval(account_state, config): def apply_gateway_failure(account_name, failures, state, config, now, admin, profile): latest = failures[-1] account_state = state.setdefault("accounts", {}).setdefault(account_name, {}) + policy = protect_policy(profile) + protected_probe = None + if policy is not None: + protected_probe = probe_account_with_protection(profile, config, "gateway-failure-confirm") + protected_probe["sourceGatewayFailure"] = { + "requestId": latest.get("requestId"), + "clientRequestId": latest.get("clientRequestId"), + "failureKind": latest.get("failureKind"), + "path": latest.get("path"), + "countInRun": len(failures), + } + account_state["lastProbeAt"] = iso(now) + account_state["lastProbe"] = { + "ok": protected_probe.get("ok"), + "purpose": protected_probe.get("purpose"), + "trustUpstream": protected_probe.get("trustUpstream"), + "sentinelProtect": protected_probe.get("sentinelProtect"), + "successMaxIntervalMinutes": success_max_interval(profile, config), + "httpStatus": protected_probe.get("httpStatus"), + "durationMs": protected_probe.get("durationMs"), + "markerMatched": protected_probe.get("markerMatched"), + "transportOk": protected_probe.get("transportOk"), + "outputHash": protected_probe.get("outputHash"), + "outputPreview": protected_probe.get("outputPreview"), + "responseBodyHash": protected_probe.get("responseBodyHash"), + "responseBodyPreview": protected_probe.get("responseBodyPreview"), + "error": protected_probe.get("error"), + "errorDetails": protected_probe.get("errorDetails"), + "usage": protected_probe.get("usage"), + "failureKind": protected_probe.get("failureKind"), + "sdk": protected_probe.get("sdk"), + "requestShape": protected_probe.get("requestShape"), + "action": {"taken": False, "type": "protect-confirm-pass" if protected_probe.get("markerMatched") is True else "protect-confirm-fail"}, + "sourceGatewayFailure": protected_probe.get("sourceGatewayFailure"), + } + add_usage(state, account_state, now, protected_probe.get("usage") or {}) + account_state["sentinelProtect"] = protected_probe.get("sentinelProtect") + account_state["trustUpstream"] = profile.get("trustUpstream") is True + account_state["successMaxIntervalMinutes"] = success_max_interval(profile, config) + if protected_probe.get("markerMatched") is True: + interval = next_success_interval(account_state, config, profile) + account_state["successStreak"] = int(account_state.get("successStreak") or 0) + 1 + account_state["successIntervalMinutes"] = interval + account_state["nextProbeAfter"] = iso(add_minutes(now, interval, int(config["cadence"]["jitterPercent"]))) + account_state["lastOkAt"] = iso(now) + account_state["lastStatus"] = "gateway-failure-protect-confirmed-ok" + account_state["lastGatewayFailureAt"] = iso(now) + account_state["lastGatewayFailure"] = { + "accountName": account_name, + "accountId": latest.get("accountId"), + "requestId": latest.get("requestId"), + "clientRequestId": latest.get("clientRequestId"), + "failureKind": latest.get("failureKind"), + "path": latest.get("path"), + "errorPreview": latest.get("errorPreview"), + "countInRun": len(failures), + "firstAt": failures[0].get("at"), + "lastAt": latest.get("at"), + "action": {"taken": False, "type": "protect-confirm-pass"}, + "sentinelProtect": protected_probe.get("sentinelProtect"), + } + return {"taken": False, "type": "protect-confirm-pass", "sentinelProtect": protected_probe.get("sentinelProtect")} interval = next_gateway_freeze_interval(account_state, config) until = add_minutes(now, interval, int(config["freeze"]["jitterPercent"])) actions_enabled = bool(config["actions"]["enabled"]) @@ -1511,6 +1662,7 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro "countInRun": len(failures), }, "lastBadAt": iso(now), + "sentinelProtect": protected_probe.get("sentinelProtect") if isinstance(protected_probe, dict) else None, } account_state["nextProbeAfter"] = iso(until) account_state["successStreak"] = 0 @@ -1538,6 +1690,7 @@ def apply_gateway_failure(account_name, failures, state, config, now, admin, pro "intervalMinutes": interval, "freezeUntil": iso(until), "action": action, + "sentinelProtect": protected_probe.get("sentinelProtect") if isinstance(protected_probe, dict) else None, } return action @@ -1721,7 +1874,7 @@ def main(): actions = [] if (config["monitor"]["enabled"] or forced_names) and due: with ThreadPoolExecutor(max_workers=max(1, len(due))) as executor: - futures = {executor.submit(probe_account, item["profile"], config, item["purpose"]): item["profile"] for item in due} + futures = {executor.submit(probe_account_with_protection, item["profile"], config, item["purpose"]): item["profile"] for item in due} for future in as_completed(futures): result = future.result() results.append(result) @@ -1760,6 +1913,7 @@ def main(): "accountName": item.get("accountName"), "purpose": item.get("purpose"), "trustUpstream": item.get("trustUpstream"), + "sentinelProtect": item.get("sentinelProtect"), "ok": item.get("ok"), "markerMatched": item.get("markerMatched"), "httpStatus": item.get("httpStatus"), diff --git a/scripts/src/platform-infra-sub2api-codex.ts b/scripts/src/platform-infra-sub2api-codex.ts index 710a9c39..3c4ca3c8 100644 --- a/scripts/src/platform-infra-sub2api-codex.ts +++ b/scripts/src/platform-infra-sub2api-codex.ts @@ -105,6 +105,7 @@ interface CodexProfile { openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null; upstreamUserAgent: string | null; trustUpstream: boolean; + sentinelProtect: CodexSentinelProtectPolicy; priority: number; capacity: number; loadFactor: number; @@ -156,12 +157,21 @@ interface CodexPoolProfileConfig { openaiResponsesWebSocketsV2Mode: OpenAIResponsesWebSocketsV2Mode | null; upstreamUserAgent: string | null; trustUpstream: boolean; + sentinelProtect: CodexSentinelProtectPolicy; priority: number; capacity: number | null; loadFactor: number | null; tempUnschedulable: CodexTempUnschedulablePolicy; } +export interface CodexSentinelProtectPolicy { + enabled: boolean; + consecutiveFailures: number; + initialRetryDelaySeconds: number; + maxRetryDelaySeconds: number; + backoffMultiplier: number; +} + interface CodexPoolPublicExposureConfig { enabled: boolean; proxyName: string; @@ -714,10 +724,12 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi upstreamUserAgent: profile.upstreamUserAgent, openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode, trustUpstream: profile.trustUpstream, + sentinelProtect: profile.sentinelProtect, }), openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode, upstreamUserAgent: profile.upstreamUserAgent, trustUpstream: profile.trustUpstream, + sentinelProtect: profile.sentinelProtect, priority: profile.priority, capacity: profile.capacity, loadFactor: profile.loadFactor, @@ -1098,6 +1110,7 @@ function collectCodexProfiles(): CodexProfile[] { openaiResponsesWebSocketsV2Mode: entry.openaiResponsesWebSocketsV2Mode, upstreamUserAgent: entry.upstreamUserAgent, trustUpstream: entry.trustUpstream, + sentinelProtect: entry.sentinelProtect, priority: entry.priority, capacity: entry.capacity ?? pool.defaultAccountCapacity, loadFactor: entry.loadFactor ?? pool.defaultAccountLoadFactor, @@ -1171,6 +1184,7 @@ function discoverCodexProfileConfigs( openaiResponsesWebSocketsV2Mode: null, upstreamUserAgent: null, trustUpstream: false, + sentinelProtect: defaultCodexSentinelProtectPolicy(), priority: defaultPriority, capacity: null, loadFactor: null, @@ -1340,6 +1354,7 @@ function readProfileConfig( const openaiResponsesWebSocketsV2Mode = readOpenAIResponsesWebSocketsV2Mode(entry.openaiResponsesWebSocketsV2Mode, `profiles.entries[${index}].openaiResponsesWebSocketsV2Mode`); const upstreamUserAgent = readUpstreamUserAgent(entry.upstreamUserAgent, `profiles.entries[${index}].upstreamUserAgent`); const trustUpstream = readTrustUpstream(entry.trustUpstream, `profiles.entries[${index}].trustUpstream`); + const sentinelProtect = readSentinelProtectPolicy(entry.sentinelProtect, `profiles.entries[${index}].sentinelProtect`); const priority = readAccountPriority(entry.priority, `profiles.entries[${index}].priority`, defaultPriority); const capacity = entry.capacity === undefined || entry.capacity === null ? null : readAccountCapacity(entry.capacity, `profiles.entries[${index}].capacity`); const loadFactor = entry.loadFactor === undefined || entry.loadFactor === null ? null : readAccountLoadFactor(entry.loadFactor, `profiles.entries[${index}].loadFactor`); @@ -1354,6 +1369,7 @@ function readProfileConfig( openaiResponsesWebSocketsV2Mode, upstreamUserAgent, trustUpstream, + sentinelProtect, priority, capacity, loadFactor, @@ -1396,6 +1412,52 @@ function readTrustUpstream(value: unknown, key: string): boolean { return parsed; } +export function defaultCodexSentinelProtectPolicy(): CodexSentinelProtectPolicy { + return { + enabled: false, + consecutiveFailures: 1, + initialRetryDelaySeconds: 2, + maxRetryDelaySeconds: 60, + backoffMultiplier: 2, + }; +} + +function readSentinelProtectPolicy(value: unknown, key: string): CodexSentinelProtectPolicy { + const fallback = defaultCodexSentinelProtectPolicy(); + if (value === undefined || value === null) return fallback; + if (!isRecord(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML object`); + const enabled = readBooleanConfig(value.enabled, `${key}.enabled`, fallback.enabled); + if (!enabled) return { ...fallback, enabled: false }; + const required = ["consecutiveFailures", "initialRetryDelaySeconds", "maxRetryDelaySeconds", "backoffMultiplier"]; + for (const field of required) { + if (value[field] === undefined || value[field] === null) { + throw new Error(`${codexPoolConfigPath}.${key}.${field} is required when enabled=true`); + } + } + const consecutiveFailures = readBoundedInteger(value.consecutiveFailures, `${key}.consecutiveFailures`, 1, 20); + const initialRetryDelaySeconds = readBoundedInteger(value.initialRetryDelaySeconds, `${key}.initialRetryDelaySeconds`, 1, 3600); + const maxRetryDelaySeconds = readBoundedInteger(value.maxRetryDelaySeconds, `${key}.maxRetryDelaySeconds`, 1, 3600); + const backoffMultiplier = readBoundedInteger(value.backoffMultiplier, `${key}.backoffMultiplier`, 1, 10); + if (maxRetryDelaySeconds < initialRetryDelaySeconds) { + throw new Error(`${codexPoolConfigPath}.${key}.maxRetryDelaySeconds must be >= initialRetryDelaySeconds`); + } + return { + enabled: true, + consecutiveFailures, + initialRetryDelaySeconds, + maxRetryDelaySeconds, + backoffMultiplier, + }; +} + +function readBoundedInteger(value: unknown, key: string, min: number, max: number): number { + const parsed = numberValue(value); + if (parsed === null || !Number.isInteger(parsed) || parsed < min || parsed > max) { + throw new Error(`${codexPoolConfigPath}.${key} must be an integer from ${min} to ${max}`); + } + return parsed; +} + function readAccountPriority(value: unknown, key: string, fallback = defaultAccountPriority): number { if (value === undefined || value === null) return fallback; const priority = numberValue(value); @@ -1723,6 +1785,7 @@ function redactProfile(profile: CodexProfile): Record { openaiResponsesWebSocketsV2Mode: profile.openaiResponsesWebSocketsV2Mode, upstreamUserAgent: profile.upstreamUserAgent, trustUpstream: profile.trustUpstream, + sentinelProtect: profile.sentinelProtect, priority: profile.priority, capacity: profile.capacity, loadFactor: profile.loadFactor, @@ -1745,6 +1808,8 @@ function compactProfile(profile: CodexProfile): Record { model: profile.model, priority: profile.priority, trustUpstream: profile.trustUpstream, + sentinelProtectEnabled: profile.sentinelProtect.enabled, + sentinelProtectConsecutiveFailures: profile.sentinelProtect.enabled ? profile.sentinelProtect.consecutiveFailures : undefined, capacity: profile.capacity, loadFactor: profile.loadFactor, tempUnschedulableEnabled: profile.tempUnschedulable.enabled, @@ -2061,6 +2126,7 @@ function compactSentinelErrorDetails(value: unknown): Record | function compactSentinelQuarantine(item: unknown): Record { if (!isRecord(item)) return {}; + const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {}; return { accountName: item.accountName, until: item.until, @@ -2068,6 +2134,12 @@ function compactSentinelQuarantine(item: unknown): Record { reason: item.reason, failureKind: item.failureKind, intervalMinutes: item.intervalMinutes, + sentinelProtect: Object.keys(protect).length > 0 ? { + enabled: protect.enabled, + decision: protect.decision, + failureCount: protect.failureCount, + threshold: protect.threshold, + } : undefined, error: compactSentinelErrorDetails(item.errorDetails), }; } @@ -2076,6 +2148,7 @@ function compactSentinelRecentAccount(item: unknown): Record { if (!isRecord(item)) return {}; const action = isRecord(item.action) ? item.action : {}; const error = compactSentinelErrorDetails(item.errorDetails); + const protect = isRecord(item.sentinelProtect) ? item.sentinelProtect : {}; return { accountName: item.accountName, lastProbeAt: item.lastProbeAt, @@ -2086,6 +2159,13 @@ function compactSentinelRecentAccount(item: unknown): Record { httpStatus: item.httpStatus, durationMs: item.durationMs, markerMatched: item.markerMatched, + sentinelProtect: Object.keys(protect).length > 0 ? { + enabled: protect.enabled, + decision: protect.decision, + protected: protect.protected, + failureCount: protect.failureCount, + threshold: protect.threshold, + } : undefined, failureKind: item.failureKind, requestShape: item.requestShape, action: Object.keys(action).length > 0 ? { @@ -2238,6 +2318,7 @@ function compactSentinelProbeResult(parsed: Record | null): Rec "purpose", "ok", "markerMatched", + "sentinelProtect", "httpStatus", "durationMs", "usage", @@ -2307,12 +2388,14 @@ function renderSentinelReport( lines.push(""); lines.push("ACCOUNTS"); lines.push(renderTable([ - ["ACCOUNT", "STATE", "Q", "T", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"], + ["ACCOUNT", "STATE", "Q", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"], ...accounts.map((account) => [ stringValue(account.account) ?? "-", stringValue(account.status) ?? "-", account.quarantineActive === true ? "Y" : "-", account.trustUpstream === true ? "Y" : account.trustUpstream === false ? "N" : "-", + account.sentinelProtectEnabled === true ? textValue(account.sentinelProtectThreshold) : "-", + account.sentinelProtectDecision ? `${textValue(account.sentinelProtectFailureCount)}/${textValue(account.sentinelProtectThreshold)}` : "-", textValue(account.freezeIntervalMin), textValue(account.successIntervalMin), textValue(account.successMaxIntervalMin), @@ -2346,7 +2429,7 @@ function renderSentinelReport( ])); } lines.push(""); - lines.push("LEGEND Q=quarantined T=trusted upstream M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions"); + lines.push("LEGEND Q=quarantined T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions"); lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw"); return lines.join("\n"); } @@ -2687,6 +2770,7 @@ function sentinelProfileSecrets(profiles: CodexProfile[]): CodexPoolSentinelProf apiKey: profile.apiKey ?? "", upstreamUserAgent: profile.upstreamUserAgent, trustUpstream: profile.trustUpstream, + sentinelProtect: profile.sentinelProtect, })); } @@ -3389,6 +3473,7 @@ export function codexPoolSentinelProbeConfigFingerprint(input: { upstreamUserAgent: string | null; openaiResponsesWebSocketsV2Mode: string | null; trustUpstream: boolean; + sentinelProtect: CodexSentinelProtectPolicy; }): string { return fingerprint(JSON.stringify({ accountName: input.accountName, @@ -3398,6 +3483,7 @@ export function codexPoolSentinelProbeConfigFingerprint(input: { upstreamUserAgent: input.upstreamUserAgent, openaiResponsesWebSocketsV2Mode: input.openaiResponsesWebSocketsV2Mode, trustUpstream: input.trustUpstream, + sentinelProtect: input.sentinelProtect, })); } @@ -3517,6 +3603,24 @@ def error_code(probe): return openai_error.get("code") or openai_error.get("type") or "" return details.get("kind") or "" +def sentinel_protect_summary(account_state, probe): + probe_protect = probe.get("sentinelProtect") if isinstance(probe.get("sentinelProtect"), dict) else {} + config_protect = account_state.get("sentinelProtectConfig") if isinstance(account_state.get("sentinelProtectConfig"), dict) else {} + source = probe_protect if probe_protect else config_protect + if not isinstance(source, dict) or source.get("enabled") is not True: + return { + "enabled": False, + "threshold": None, + "decision": None, + "failureCount": None, + } + return { + "enabled": True, + "threshold": source.get("threshold") or source.get("consecutiveFailures"), + "decision": probe_protect.get("decision"), + "failureCount": probe_protect.get("failureCount"), + } + def report(): cronjob, cron_error = kube_json(["-n", NAMESPACE, "get", "cronjob", CRONJOB_NAME]) state_cm, state_error = kube_json(["-n", NAMESPACE, "get", "configmap", STATE_NAME]) @@ -3544,6 +3648,7 @@ def report(): if gateway_failure and (not account_state.get("lastProbeAt") or str(account_state.get("lastGatewayFailureAt") or "") >= str(account_state.get("lastProbeAt") or "")): last_action = action_type(gateway_failure) ledger = account_ledger(account_state) + protect = sentinel_protect_summary(account_state, probe) account_rows.append({ "account": name, "status": account_state.get("lastStatus"), @@ -3566,6 +3671,10 @@ def report(): "lastPurpose": probe.get("purpose"), "lastHttp": probe.get("httpStatus"), "lastMarker": probe.get("markerMatched"), + "sentinelProtectEnabled": protect.get("enabled"), + "sentinelProtectDecision": protect.get("decision"), + "sentinelProtectFailureCount": protect.get("failureCount"), + "sentinelProtectThreshold": protect.get("threshold"), "lastFailureKind": last_failure_kind, "lastErrorCode": error_code(probe), "lastAction": last_action, @@ -3987,6 +4096,8 @@ def sentinel_probe_change_reasons(current, profile): runtime_ws_mode = empty_to_none(extra.get("openai_apikey_responses_websockets_v2_mode")) expected_trust_upstream = profile.get("trustUpstream") is True runtime_trust_upstream = extra.get("unidesk_trust_upstream") is True + expected_protect = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} + runtime_protect = extra.get("unidesk_sentinel_protect") if isinstance(extra.get("unidesk_sentinel_protect"), dict) else {"enabled": False} reasons = [] if empty_to_none(extra.get("unidesk_codex_profile")) != profile.get("profile"): reasons.append("profile") @@ -4000,6 +4111,8 @@ def sentinel_probe_change_reasons(current, profile): reasons.append("responses-websockets-v2-mode") if runtime_trust_upstream != expected_trust_upstream: reasons.append("trust-upstream") + if runtime_protect != expected_protect: + reasons.append("sentinel-protect") return reasons def curl_api(method, path, bearer=None, payload=None): @@ -4214,6 +4327,7 @@ def account_payload(profile, group_id): "unidesk_codex_profile": profile["profile"], "unidesk_managed": True, "unidesk_trust_upstream": profile.get("trustUpstream") is True, + "unidesk_sentinel_protect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, } ws_mode = profile.get("openaiResponsesWebSocketsV2Mode") if ws_mode: @@ -4264,6 +4378,7 @@ def planned_sentinel_account_results(profiles, existing_accounts): "profileConfig": { "accountName": profile["accountName"], "trustUpstream": profile.get("trustUpstream") is True, + "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, }, "sentinelProbeConfigFingerprint": profile.get("sentinelProbeConfigFingerprint"), "sentinelProbeRequired": quality_gate_required, @@ -4308,6 +4423,7 @@ def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_fr "profileConfig": { "accountName": profile["accountName"], "trustUpstream": profile.get("trustUpstream") is True, + "sentinelProtect": profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False}, }, "accountId": data.get("id") if isinstance(data, dict) else None, "action": action, @@ -4467,6 +4583,7 @@ def sentinel_runtime_status(): "failureKind": quarantine.get("failureKind"), "errorDetails": quarantine.get("errorDetails"), "intervalMinutes": quarantine.get("intervalMinutes"), + "sentinelProtect": quarantine.get("sentinelProtect"), }) last_probe = account_state.get("lastProbe") if isinstance(last_probe, dict): @@ -4480,6 +4597,7 @@ def sentinel_runtime_status(): "httpStatus": last_probe.get("httpStatus"), "durationMs": last_probe.get("durationMs"), "markerMatched": last_probe.get("markerMatched"), + "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), "outputHash": last_probe.get("outputHash"), "outputPreview": last_probe.get("outputPreview"), "responseBodyHash": last_probe.get("responseBodyHash"), @@ -4652,6 +4770,7 @@ def clamp_sentinel_success_cadence_for_config(state, profiles, now): quarantine = account_state.get("quarantine") if isinstance(quarantine, dict) and quarantine.get("active") is True: account_state["trustUpstream"] = profile.get("trustUpstream") is True + account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile) continue try: @@ -4661,6 +4780,7 @@ def clamp_sentinel_success_cadence_for_config(state, profiles, now): next_epoch = parse_iso_epoch(account_state.get("nextProbeAfter")) max_interval = profile_success_max_interval(profile) account_state["trustUpstream"] = profile.get("trustUpstream") is True + account_state["sentinelProtectConfig"] = profile.get("sentinelProtect") if isinstance(profile.get("sentinelProtect"), dict) else {"enabled": False} account_state["successMaxIntervalMinutes"] = max_interval if interval <= max_interval and (next_epoch is None or next_epoch <= now_epoch + max_interval * 60): continue @@ -4751,6 +4871,7 @@ def ensure_sentinel_state_for_sync(account_results, pending_only=False): account_state["successIntervalMinutes"] = 0 profile_config = item.get("profileConfig") if isinstance(item.get("profileConfig"), dict) else {} account_state["trustUpstream"] = profile_config.get("trustUpstream") is True + account_state["sentinelProtectConfig"] = profile_config.get("sentinelProtect") if isinstance(profile_config.get("sentinelProtect"), dict) else {"enabled": False} account_state["successMaxIntervalMinutes"] = profile_success_max_interval(profile_config) account_state["lastStatus"] = "pending-sentinel-quality-gate" account_state["qualityGate"] = { @@ -4811,6 +4932,7 @@ def sentinel_state_summary(): "failureKind": quarantine.get("failureKind"), "errorDetails": quarantine.get("errorDetails"), "intervalMinutes": quarantine.get("intervalMinutes"), + "sentinelProtect": quarantine.get("sentinelProtect"), }) last_probe = account_state.get("lastProbe") if isinstance(last_probe, dict): @@ -4824,6 +4946,7 @@ def sentinel_state_summary(): "httpStatus": last_probe.get("httpStatus"), "durationMs": last_probe.get("durationMs"), "markerMatched": last_probe.get("markerMatched"), + "sentinelProtect": last_probe.get("sentinelProtect") or account_state.get("sentinelProtectConfig"), "usage": last_probe.get("usage"), "responseBodyHash": last_probe.get("responseBodyHash"), "responseBodyPreview": last_probe.get("responseBodyPreview"),