fix: reduce Sub2API compact threshold and observe failures

This commit is contained in:
Codex
2026-06-12 14:41:58 +00:00
parent b41847853a
commit 9ab1acbe1d
4 changed files with 55 additions and 3 deletions
+2
View File
@@ -189,6 +189,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm
-`platform-infra/<apiKeySecretName>.<apiKeySecretKey>` 读取统一 API key。
- 把当前 `~/.codex/config.toml``~/.codex/auth.json` 备份为 `.<backupSuffix>`,默认 `.pre-sub2api`
- 重写默认 `~/.codex` 消费端,固定指向 `https://sub2api.74-48-78-17.nip.io/`provider 名称和 wire API 来自 `localCodex`
-`localCodex.modelContextWindow` / `localCodex.modelAutoCompactTokenLimit` 写入 `model_context_window` / `model_auto_compact_token_limit`,用于统一控制 Codex auto compact 触发窗口,避免 GPT-5.5 消费端生成过大的 `/responses/compact` 长请求。
-`localCodex` 写入 Codex transport 标记:`supports_websockets``[features] responses_websockets_v2` 必须同开同关。只有至少一个上游通过 direct Codex WSv2 probe 时才启用;否则保持 HTTP Responses,避免每次原入口先经历无效 WS reconnect。
- 用统一 key 做一次 gateway 验证。
@@ -209,6 +210,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm
## 排障
- Codex pool 哨兵、账号冻结/恢复、marker-only 判断或 probe 周期看不清:第一步跑 `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report`。这个报表是主观察面;只有报表缺字段或需要底层证据时,才继续看 `--raw`、CronJob log、state ConfigMap 或 Sub2API 管理 UI。若看到“临时不可调度状态”且包含规则序号/匹配关键词,检查 Sub2API `account_temp_unschedulable` 日志和账号 `temp_unschedulable_*` 字段;sentinel 只解释 `schedulable=false` 的 active quarantine,不解释这类内置临时冷却。
- 只加强监控、不让哨兵自动冻结账号时,把 YAML `sentinel.actions.enabled=false``codex-pool sync --confirm`。此时 marker probe 和 gateway failure monitor 仍记录 `would-freeze` / observe-only 证据,但不会通过 Sub2API admin 写 `schedulable=false``/responses/compact``codex.remote_compact.failed` 和 compact 上游 5xx failover 只作为 `gateway-compact-*` 观察事件记录,不作为哨兵自动切换触发器。
- 单个 request id 报 502/503/中断/没有自动切号:第一步跑 `bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <requestId>`。先看 `outcome``reason``FAILOVER``SELECT-FAILED``ACCOUNT SIGNALS``WINDOW STATS`;只有 trace 报表缺字段或需要审计原始日志时,才加 `--show-lines``--raw`。若 `reason=failover-attempted-no-candidate`,说明切号动作已发生,但 scheduler 在排除失败账号后没有可用候选;继续用 `sentinel-report``validate --full` 区分 sentinel quarantine、request-path temp-unschedulable、账号 status 或容量耗尽。
- profile invalid:先修 `~/.codex/config.toml.<profile>``base_url``wire_api``model``auth.json.<profile>` 的 API key;不要在 YAML 中写密钥。
- Sub2API 卡在 `wait-postgres` / `wait-redis` 或服务内大量 `context deadline exceeded`:先跑 `sub2api status``networkPolicy.ok`,再跑 `sub2api validate``postgresCrossPodPgIsReady` / `redisCrossPodPing`;缺失或异常时用 `sub2api apply --confirm` 恢复受控 `NetworkPolicy/allow-all`,不要保留手工 iptables bypass 作为长期修复。
@@ -152,6 +152,8 @@ localCodex:
backupSuffix: pre-sub2api
providerName: OpenAI
wireApi: responses
modelContextWindow: 272000
modelAutoCompactTokenLimit: 240000
supportsWebSockets: false
responsesWebSocketsV2: false
responsesSmokeModel: gpt-5.5
@@ -159,7 +161,7 @@ sentinel:
monitor:
enabled: true
actions:
enabled: true
enabled: false
schedule: "*/1 * * * *"
image: python:3.12-alpine
sdk:
@@ -1346,13 +1346,25 @@ def gateway_monitor_paths(config):
return {"/responses", "/v1/responses", "/responses/compact", "/v1/responses/compact"}
def gateway_failure_kind(message, payload, config):
if "openai.forward_failed" not in message or not isinstance(payload, dict):
if not isinstance(payload, dict):
return None
path = payload.get("path")
if path not in gateway_monitor_paths(config):
return None
if payload.get("account_id") is None:
return None
if "codex.remote_compact.failed" in message:
status = payload.get("status_code")
if isinstance(status, int) and status >= 500:
return "gateway-compact-final-failure"
return None
if "openai.upstream_failover_switching" in message and path in ("/responses/compact", "/v1/responses/compact"):
upstream_status = payload.get("upstream_status")
if isinstance(upstream_status, int) and upstream_status >= 500:
return "gateway-compact-upstream-failover"
return None
if "openai.forward_failed" not in message:
return None
error_text = str(payload.get("error") or "").lower()
fallback_written = payload.get("fallback_error_response_written") is True
upstream_already_written = payload.get("upstream_error_response_already_written") is True
@@ -1404,7 +1416,7 @@ def gateway_failure_kind(message, payload, config):
return None
def gateway_failure_is_observe_only(failure_kind):
return failure_kind in {"gateway-session-affinity-failure"}
return failure_kind in {"gateway-session-affinity-failure", "gateway-compact-final-failure", "gateway-compact-upstream-failover"}
def gateway_failure_item(ts, pod_name, payload, failure_kind):
request_id = payload.get("request_id") or sha(json.dumps(payload, sort_keys=True, ensure_ascii=False))
@@ -202,6 +202,8 @@ interface CodexPoolLocalCodexConfig {
backupSuffix: string;
providerName: string;
wireApi: string;
modelContextWindow: number;
modelAutoCompactTokenLimit: number;
supportsWebSockets: boolean;
responsesWebSocketsV2: boolean;
responsesSmokeModel: string;
@@ -211,6 +213,8 @@ interface CodexLocalConsumerTomlOptions {
providerName: string;
baseUrl: string;
wireApi: string;
modelContextWindow: number;
modelAutoCompactTokenLimit: number;
supportsWebSockets: boolean;
responsesWebSocketsV2: boolean;
}
@@ -1009,6 +1013,8 @@ async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOp
baseUrl: codexConsumerBaseUrl(pool),
providerName: pool.localCodex.providerName,
wireApi: pool.localCodex.wireApi,
modelContextWindow: pool.localCodex.modelContextWindow,
modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit,
supportsWebSockets: pool.localCodex.supportsWebSockets,
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
responsesSmokeModel: pool.localCodex.responsesSmokeModel,
@@ -1274,6 +1280,8 @@ function defaultCodexPoolConfig(): CodexPoolConfig {
backupSuffix: "pre-sub2api",
providerName: "OpenAI",
wireApi: "responses",
modelContextWindow: 272000,
modelAutoCompactTokenLimit: 240000,
supportsWebSockets: true,
responsesWebSocketsV2: true,
responsesSmokeModel: "gpt-5.5",
@@ -1444,6 +1452,11 @@ function readPositiveInteger(value: unknown, key: string): number {
return parsed;
}
function readPositiveIntegerConfig(value: unknown, key: string, fallback: number): number {
if (value === undefined || value === null) return fallback;
return readPositiveInteger(value, key);
}
function readCaddyRetryMethods(value: unknown, key: string): string[] {
if (value === undefined || value === null) return [];
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
@@ -1619,6 +1632,8 @@ function readLocalCodexConfig(value: unknown, defaults: CodexPoolLocalCodexConfi
backupSuffix: stringValue(value.backupSuffix) ?? defaults.backupSuffix,
providerName: stringValue(value.providerName) ?? defaults.providerName,
wireApi: stringValue(value.wireApi) ?? defaults.wireApi,
modelContextWindow: readPositiveIntegerConfig(value.modelContextWindow, "localCodex.modelContextWindow", defaults.modelContextWindow),
modelAutoCompactTokenLimit: readPositiveIntegerConfig(value.modelAutoCompactTokenLimit, "localCodex.modelAutoCompactTokenLimit", defaults.modelAutoCompactTokenLimit),
supportsWebSockets: readBooleanConfig(value.supportsWebSockets, "localCodex.supportsWebSockets", defaults.supportsWebSockets),
responsesWebSocketsV2: readBooleanConfig(value.responsesWebSocketsV2, "localCodex.responsesWebSocketsV2", defaults.responsesWebSocketsV2),
responsesSmokeModel: stringValue(value.responsesSmokeModel) ?? defaults.responsesSmokeModel,
@@ -3097,6 +3112,8 @@ function writeLocalCodexConfig(pool: CodexPoolConfig, apiKey: string): Record<st
name: pool.localCodex.providerName,
baseUrl: codexConsumerBaseUrl(pool),
wireApi: pool.localCodex.wireApi,
modelContextWindow: pool.localCodex.modelContextWindow,
modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit,
supportsWebSockets: pool.localCodex.supportsWebSockets,
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
},
@@ -3116,6 +3133,8 @@ function updateCodexConfigToml(current: string, pool: CodexPoolConfig): string {
providerName: pool.localCodex.providerName,
baseUrl: codexConsumerBaseUrl(pool),
wireApi: pool.localCodex.wireApi,
modelContextWindow: pool.localCodex.modelContextWindow,
modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit,
supportsWebSockets: pool.localCodex.supportsWebSockets,
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
});
@@ -3127,6 +3146,8 @@ function codexConsumerBaseUrl(pool: CodexPoolConfig): string {
export function renderCodexLocalConsumerToml(current: string, options: CodexLocalConsumerTomlOptions): string {
let next = upsertTopLevelTomlString(current, "model_provider", options.providerName);
next = upsertTopLevelTomlInteger(next, "model_context_window", options.modelContextWindow);
next = upsertTopLevelTomlInteger(next, "model_auto_compact_token_limit", options.modelAutoCompactTokenLimit);
next = upsertProviderSection(next, options);
next = upsertTomlSectionBoolean(next, "features", "responses_websockets_v2", options.responsesWebSocketsV2);
return next.endsWith("\n") ? next : `${next}\n`;
@@ -3146,6 +3167,21 @@ function upsertTopLevelTomlString(text: string, key: string, value: string): str
return lines.join("\n");
}
function upsertTopLevelTomlInteger(text: string, key: string, value: number): string {
const lines = text.split(/\r?\n/u);
const firstSection = lines.findIndex((line) => /^\s*\[/.test(line));
const end = firstSection === -1 ? lines.length : firstSection;
const assignment = `${key} = ${value}`;
for (let index = 0; index < end; index += 1) {
if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) {
lines[index] = assignment;
return lines.join("\n");
}
}
lines.splice(end, 0, assignment);
return lines.join("\n");
}
function upsertProviderSection(text: string, options: CodexLocalConsumerTomlOptions): string {
const { providerName, baseUrl, wireApi, supportsWebSockets } = options;
const header = `[model_providers.${providerName}]`;