fix: reduce Sub2API compact threshold and observe failures
This commit is contained in:
@@ -189,6 +189,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm
|
||||
- 从 `platform-infra/<apiKeySecretName>.<apiKeySecretKey>` 读取统一 API key。
|
||||
- 把当前 `~/.codex/config.toml` 和 `~/.codex/auth.json` 备份为 `.<backupSuffix>`,默认 `.pre-sub2api`。
|
||||
- 重写默认 `~/.codex` 消费端,固定指向 `https://sub2api.74-48-78-17.nip.io/`,provider 名称和 wire API 来自 `localCodex`。
|
||||
- 按 `localCodex.modelContextWindow` / `localCodex.modelAutoCompactTokenLimit` 写入 `model_context_window` / `model_auto_compact_token_limit`,用于统一控制 Codex auto compact 触发窗口,避免 GPT-5.5 消费端生成过大的 `/responses/compact` 长请求。
|
||||
- 按 `localCodex` 写入 Codex transport 标记:`supports_websockets` 与 `[features] responses_websockets_v2` 必须同开同关。只有至少一个上游通过 direct Codex WSv2 probe 时才启用;否则保持 HTTP Responses,避免每次原入口先经历无效 WS reconnect。
|
||||
- 用统一 key 做一次 gateway 验证。
|
||||
|
||||
@@ -209,6 +210,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm
|
||||
## 排障
|
||||
|
||||
- Codex pool 哨兵、账号冻结/恢复、marker-only 判断或 probe 周期看不清:第一步跑 `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report`。这个报表是主观察面;只有报表缺字段或需要底层证据时,才继续看 `--raw`、CronJob log、state ConfigMap 或 Sub2API 管理 UI。若看到“临时不可调度状态”且包含规则序号/匹配关键词,检查 Sub2API `account_temp_unschedulable` 日志和账号 `temp_unschedulable_*` 字段;sentinel 只解释 `schedulable=false` 的 active quarantine,不解释这类内置临时冷却。
|
||||
- 只加强监控、不让哨兵自动冻结账号时,把 YAML `sentinel.actions.enabled=false` 后 `codex-pool sync --confirm`。此时 marker probe 和 gateway failure monitor 仍记录 `would-freeze` / observe-only 证据,但不会通过 Sub2API admin 写 `schedulable=false`;`/responses/compact` 的 `codex.remote_compact.failed` 和 compact 上游 5xx failover 只作为 `gateway-compact-*` 观察事件记录,不作为哨兵自动切换触发器。
|
||||
- 单个 request id 报 502/503/中断/没有自动切号:第一步跑 `bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <requestId>`。先看 `outcome`、`reason`、`FAILOVER`、`SELECT-FAILED`、`ACCOUNT SIGNALS` 和 `WINDOW STATS`;只有 trace 报表缺字段或需要审计原始日志时,才加 `--show-lines` 或 `--raw`。若 `reason=failover-attempted-no-candidate`,说明切号动作已发生,但 scheduler 在排除失败账号后没有可用候选;继续用 `sentinel-report` 和 `validate --full` 区分 sentinel quarantine、request-path temp-unschedulable、账号 status 或容量耗尽。
|
||||
- profile invalid:先修 `~/.codex/config.toml.<profile>` 的 `base_url`、`wire_api`、`model` 或 `auth.json.<profile>` 的 API key;不要在 YAML 中写密钥。
|
||||
- Sub2API 卡在 `wait-postgres` / `wait-redis` 或服务内大量 `context deadline exceeded`:先跑 `sub2api status` 看 `networkPolicy.ok`,再跑 `sub2api validate` 看 `postgresCrossPodPgIsReady` / `redisCrossPodPing`;缺失或异常时用 `sub2api apply --confirm` 恢复受控 `NetworkPolicy/allow-all`,不要保留手工 iptables bypass 作为长期修复。
|
||||
|
||||
@@ -152,6 +152,8 @@ localCodex:
|
||||
backupSuffix: pre-sub2api
|
||||
providerName: OpenAI
|
||||
wireApi: responses
|
||||
modelContextWindow: 272000
|
||||
modelAutoCompactTokenLimit: 240000
|
||||
supportsWebSockets: false
|
||||
responsesWebSocketsV2: false
|
||||
responsesSmokeModel: gpt-5.5
|
||||
@@ -159,7 +161,7 @@ sentinel:
|
||||
monitor:
|
||||
enabled: true
|
||||
actions:
|
||||
enabled: true
|
||||
enabled: false
|
||||
schedule: "*/1 * * * *"
|
||||
image: python:3.12-alpine
|
||||
sdk:
|
||||
|
||||
@@ -1346,13 +1346,25 @@ def gateway_monitor_paths(config):
|
||||
return {"/responses", "/v1/responses", "/responses/compact", "/v1/responses/compact"}
|
||||
|
||||
def gateway_failure_kind(message, payload, config):
|
||||
if "openai.forward_failed" not in message or not isinstance(payload, dict):
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
path = payload.get("path")
|
||||
if path not in gateway_monitor_paths(config):
|
||||
return None
|
||||
if payload.get("account_id") is None:
|
||||
return None
|
||||
if "codex.remote_compact.failed" in message:
|
||||
status = payload.get("status_code")
|
||||
if isinstance(status, int) and status >= 500:
|
||||
return "gateway-compact-final-failure"
|
||||
return None
|
||||
if "openai.upstream_failover_switching" in message and path in ("/responses/compact", "/v1/responses/compact"):
|
||||
upstream_status = payload.get("upstream_status")
|
||||
if isinstance(upstream_status, int) and upstream_status >= 500:
|
||||
return "gateway-compact-upstream-failover"
|
||||
return None
|
||||
if "openai.forward_failed" not in message:
|
||||
return None
|
||||
error_text = str(payload.get("error") or "").lower()
|
||||
fallback_written = payload.get("fallback_error_response_written") is True
|
||||
upstream_already_written = payload.get("upstream_error_response_already_written") is True
|
||||
@@ -1404,7 +1416,7 @@ def gateway_failure_kind(message, payload, config):
|
||||
return None
|
||||
|
||||
def gateway_failure_is_observe_only(failure_kind):
|
||||
return failure_kind in {"gateway-session-affinity-failure"}
|
||||
return failure_kind in {"gateway-session-affinity-failure", "gateway-compact-final-failure", "gateway-compact-upstream-failover"}
|
||||
|
||||
def gateway_failure_item(ts, pod_name, payload, failure_kind):
|
||||
request_id = payload.get("request_id") or sha(json.dumps(payload, sort_keys=True, ensure_ascii=False))
|
||||
|
||||
@@ -202,6 +202,8 @@ interface CodexPoolLocalCodexConfig {
|
||||
backupSuffix: string;
|
||||
providerName: string;
|
||||
wireApi: string;
|
||||
modelContextWindow: number;
|
||||
modelAutoCompactTokenLimit: number;
|
||||
supportsWebSockets: boolean;
|
||||
responsesWebSocketsV2: boolean;
|
||||
responsesSmokeModel: string;
|
||||
@@ -211,6 +213,8 @@ interface CodexLocalConsumerTomlOptions {
|
||||
providerName: string;
|
||||
baseUrl: string;
|
||||
wireApi: string;
|
||||
modelContextWindow: number;
|
||||
modelAutoCompactTokenLimit: number;
|
||||
supportsWebSockets: boolean;
|
||||
responsesWebSocketsV2: boolean;
|
||||
}
|
||||
@@ -1009,6 +1013,8 @@ async function codexPoolConfigureLocal(config: UniDeskConfig, options: ConfirmOp
|
||||
baseUrl: codexConsumerBaseUrl(pool),
|
||||
providerName: pool.localCodex.providerName,
|
||||
wireApi: pool.localCodex.wireApi,
|
||||
modelContextWindow: pool.localCodex.modelContextWindow,
|
||||
modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit,
|
||||
supportsWebSockets: pool.localCodex.supportsWebSockets,
|
||||
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
|
||||
responsesSmokeModel: pool.localCodex.responsesSmokeModel,
|
||||
@@ -1274,6 +1280,8 @@ function defaultCodexPoolConfig(): CodexPoolConfig {
|
||||
backupSuffix: "pre-sub2api",
|
||||
providerName: "OpenAI",
|
||||
wireApi: "responses",
|
||||
modelContextWindow: 272000,
|
||||
modelAutoCompactTokenLimit: 240000,
|
||||
supportsWebSockets: true,
|
||||
responsesWebSocketsV2: true,
|
||||
responsesSmokeModel: "gpt-5.5",
|
||||
@@ -1444,6 +1452,11 @@ function readPositiveInteger(value: unknown, key: string): number {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function readPositiveIntegerConfig(value: unknown, key: string, fallback: number): number {
|
||||
if (value === undefined || value === null) return fallback;
|
||||
return readPositiveInteger(value, key);
|
||||
}
|
||||
|
||||
function readCaddyRetryMethods(value: unknown, key: string): string[] {
|
||||
if (value === undefined || value === null) return [];
|
||||
if (!Array.isArray(value)) throw new Error(`${codexPoolConfigPath}.${key} must be a YAML array`);
|
||||
@@ -1619,6 +1632,8 @@ function readLocalCodexConfig(value: unknown, defaults: CodexPoolLocalCodexConfi
|
||||
backupSuffix: stringValue(value.backupSuffix) ?? defaults.backupSuffix,
|
||||
providerName: stringValue(value.providerName) ?? defaults.providerName,
|
||||
wireApi: stringValue(value.wireApi) ?? defaults.wireApi,
|
||||
modelContextWindow: readPositiveIntegerConfig(value.modelContextWindow, "localCodex.modelContextWindow", defaults.modelContextWindow),
|
||||
modelAutoCompactTokenLimit: readPositiveIntegerConfig(value.modelAutoCompactTokenLimit, "localCodex.modelAutoCompactTokenLimit", defaults.modelAutoCompactTokenLimit),
|
||||
supportsWebSockets: readBooleanConfig(value.supportsWebSockets, "localCodex.supportsWebSockets", defaults.supportsWebSockets),
|
||||
responsesWebSocketsV2: readBooleanConfig(value.responsesWebSocketsV2, "localCodex.responsesWebSocketsV2", defaults.responsesWebSocketsV2),
|
||||
responsesSmokeModel: stringValue(value.responsesSmokeModel) ?? defaults.responsesSmokeModel,
|
||||
@@ -3097,6 +3112,8 @@ function writeLocalCodexConfig(pool: CodexPoolConfig, apiKey: string): Record<st
|
||||
name: pool.localCodex.providerName,
|
||||
baseUrl: codexConsumerBaseUrl(pool),
|
||||
wireApi: pool.localCodex.wireApi,
|
||||
modelContextWindow: pool.localCodex.modelContextWindow,
|
||||
modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit,
|
||||
supportsWebSockets: pool.localCodex.supportsWebSockets,
|
||||
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
|
||||
},
|
||||
@@ -3116,6 +3133,8 @@ function updateCodexConfigToml(current: string, pool: CodexPoolConfig): string {
|
||||
providerName: pool.localCodex.providerName,
|
||||
baseUrl: codexConsumerBaseUrl(pool),
|
||||
wireApi: pool.localCodex.wireApi,
|
||||
modelContextWindow: pool.localCodex.modelContextWindow,
|
||||
modelAutoCompactTokenLimit: pool.localCodex.modelAutoCompactTokenLimit,
|
||||
supportsWebSockets: pool.localCodex.supportsWebSockets,
|
||||
responsesWebSocketsV2: pool.localCodex.responsesWebSocketsV2,
|
||||
});
|
||||
@@ -3127,6 +3146,8 @@ function codexConsumerBaseUrl(pool: CodexPoolConfig): string {
|
||||
|
||||
export function renderCodexLocalConsumerToml(current: string, options: CodexLocalConsumerTomlOptions): string {
|
||||
let next = upsertTopLevelTomlString(current, "model_provider", options.providerName);
|
||||
next = upsertTopLevelTomlInteger(next, "model_context_window", options.modelContextWindow);
|
||||
next = upsertTopLevelTomlInteger(next, "model_auto_compact_token_limit", options.modelAutoCompactTokenLimit);
|
||||
next = upsertProviderSection(next, options);
|
||||
next = upsertTomlSectionBoolean(next, "features", "responses_websockets_v2", options.responsesWebSocketsV2);
|
||||
return next.endsWith("\n") ? next : `${next}\n`;
|
||||
@@ -3146,6 +3167,21 @@ function upsertTopLevelTomlString(text: string, key: string, value: string): str
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function upsertTopLevelTomlInteger(text: string, key: string, value: number): string {
|
||||
const lines = text.split(/\r?\n/u);
|
||||
const firstSection = lines.findIndex((line) => /^\s*\[/.test(line));
|
||||
const end = firstSection === -1 ? lines.length : firstSection;
|
||||
const assignment = `${key} = ${value}`;
|
||||
for (let index = 0; index < end; index += 1) {
|
||||
if (new RegExp(`^\\s*${escapeRegExp(key)}\\s*=`).test(lines[index])) {
|
||||
lines[index] = assignment;
|
||||
return lines.join("\n");
|
||||
}
|
||||
}
|
||||
lines.splice(end, 0, assignment);
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function upsertProviderSection(text: string, options: CodexLocalConsumerTomlOptions): string {
|
||||
const { providerName, baseUrl, wireApi, supportsWebSockets } = options;
|
||||
const header = `[model_providers.${providerName}]`;
|
||||
|
||||
Reference in New Issue
Block a user