From a91238c2006ce757c6096920df93f9c122fbec87 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 06:40:02 +0200 Subject: [PATCH 1/4] feat: restore parallel ops and diagnostics changes --- .agents/skills/unidesk-cicd/SKILL.md | 7 +- .agents/skills/unidesk-sub2api/SKILL.md | 11 +- .../skills/unidesk-sub2api/agents/openai.yaml | 4 +- .../unidesk-sub2api/references/codex-pool.md | 55 +- .../references/troubleshooting-accounts.md | 34 + config/platform-infra/sub2api-codex-pool.yaml | 22 +- .../R4.4.2_Task_Report.md | 21 + .../R6_Task_Report.md | 22 + .../R1_Task_Report.md | 37 + .../R2.1_Task_Report.md | 55 + .../R2.2.1_Task_Report.md | 107 ++ .../R2.2.2_Task_Report.md | 64 + .../R2.2.3_Task_Report.md | 53 + .../R2.2.4_Task_Report.md | 104 ++ .../pr-merge-driven-automatic-delivery.md | 2 +- docs/MDTODO/sub2api-upstream-reliability.md | 37 + docs/reference/platform-infra.md | 20 +- scripts/src/output.ts | 4 +- .../remote-python-sync.ts | 126 +- .../platform-infra-sub2api-codex/render.ts | 29 +- .../platform-infra-sub2api-codex/runtime.ts | 1282 ++++++++++++++++- .../src/platform-infra-sub2api-codex/types.ts | 6 +- 22 files changed, 2039 insertions(+), 63 deletions(-) create mode 100644 docs/MDTODO/details/observability-trace-reliability/R4.4.2_Task_Report.md create mode 100644 docs/MDTODO/details/pr-merge-driven-automatic-delivery/R6_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R1_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.1_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.1_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.2_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.3_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.4_Task_Report.md create mode 100644 docs/MDTODO/sub2api-upstream-reliability.md diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 299abf2a..0ad166f6 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -128,12 +128,13 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - CI/CD validation 阶段只能验证部署对象的 `/health` 端点和必要 provenance；禁止在 CI/CD gate 中运行 web-probe、Playwright、远程浏览器截图、用户路径 E2E 或等价重型业务探针。业务/用户入口验证只能作为发布后的独立 post-deploy validation 证据，不得阻塞 CI/CD 一键交付。 - 产品功能配置 schema 的 CI/CD 合同： - repo 内唯一约定路径固定为 `config/feature-config.schema.json`； - - schema 固定使用 JSON Schema Draft 2020-12，并由 lockfile 锁定的完整结构化 validator 校验；禁止用正则、浅层字段遍历或关闭整体 strict 代替； + - schema 固定使用 JSON Schema Draft 2020-12，并由 lockfile 锁定的完整结构化 validator 校验；禁止用正则、只检查顶层字段的浅层解析器或关闭整体 strict 代替； - schema 只描述产品功能配置，不得承载数据 authority、事件路径、投影实现或交付路径选择； - - 每个顶层 `properties` 成员必须声明唯一非空 `x-unidesk-feature`；一个产品功能只允许一个规范配置 property，禁止别名、反向变量或重复开关； + - schema 的每个顶层 property 是一个规范产品配置键，值可以是对象，不强制使用大写 env 命名； + - 每个顶层 property 必须声明唯一非空 `x-unidesk-feature`；同一功能只允许一个规范配置 property，禁止在 schema、renderer、Deployment env 或应用解析器中保留别名、反向变量或重复开关； - candidate snapshot 必须由正常 source-artifact 阶段从 owning YAML 或已渲染 workload 显式生成；禁止读取 runner `process.env` 作为产品配置 authority； - HWLAB、AgentRun 与 Sentinel consumer 必须复用同一 shared helper，并在正常 source-artifact/GitOps publish 路径执行，不得只挂到某个 consumer 的专属后处理； - - CI/CD 必须检查 schema 语法、嵌套配置值匹配和功能到配置 property 的一对一关系，并输出有界 typed warning；输入快照缺失使用独立 typed warning，不得与 schema 非法混淆； + - CI/CD 必须检查 schema 语法、嵌套配置值匹配和功能到配置 property 的一对一关系，并输出有界 typed warning；输入快照缺失使用独立的 `feature-config-input-missing` warning，不得与 schema 非法混淆； - schema 缺失、非法、candidate 缺失、配置不匹配、重复 property、validator/依赖异常或 OTel 导出失败时，只允许记录 warning；embedded 命令必须以 `0` 退出，PipelineRun、artifact、GitOps promote、Argo reconcile、runtime rollout 和 `/health` closeout 必须继续； - schema 校验器只读，禁止写回配置、注入默认值、删除变量、关闭功能、切换架构路径或执行任何补救操作；运行时必须原样接收本次待发布配置； - schema warning 必须实际导出 CI/CD OTel span/event；endpoint、serviceName、sampling 和 exporter timeout 只读 `config/platform-infra/pipelines-as-code.yaml#observability` 与精确 repository params，零/多 repository 匹配在 renderer 生成期 fail-closed； diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index 6c2413ac..0ecfd9cd 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -1,6 +1,9 @@ --- name: unidesk-sub2api -description: UniDesk Sub2API 平台运维技能。用户提到 Sub2API、sub2api、platform-infra sub2api、Codex pool、统一 API key、Sub2API FRP 暴露、Sub2API 管理 UI、配置 master ~/.codex 走 Sub2API、添加/删除 Codex 上游账号、校验 Sub2API /v1/models 时使用。 +description: >- + UniDesk Sub2API 平台运维技能。用户提到 Sub2API、platform-infra sub2api、Codex pool、统一 API key、 + runtime CRUD、精准批量账号操作、临时不可调度、上游错误率、客户可见错误、换号/failover、 + 模型映射、可用模型探测、FRP 暴露、管理 UI、配置 master ~/.codex、上游账号增删或校验 /v1/models 时使用。 --- # UniDesk Sub2API @@ -35,12 +38,14 @@ bun scripts/cli.ts platform-infra sub2api image-prepull --target PK01 --confirm ## 何时读取 reference - 部署、状态、target 边界、PK01 host-Docker、k3s target、egress proxy、镜像升级：读 [references/operations.md](references/operations.md)。 -- Codex pool、统一 key、trace、account temp-unschedulable、`codex-pool sync|validate`：读 [references/codex-pool.md](references/codex-pool.md)。 +- Codex pool、统一 key、runtime CRUD、精准批量、模型探测、trace、account temp-unschedulable、`codex-pool sync|validate`：读 [references/codex-pool.md](references/codex-pool.md)。 - Sentinel、marker-only 判定、账号冻结/恢复、`sentinel-report|sentinel-probe|sentinel-image`：读 [references/sentinel.md](references/sentinel.md)。 - 受保护手动账号代理、分组绑定、WebUI account test：读 [references/manual-accounts.md](references/manual-accounts.md)。 - 添加或删除上游 profile/account：读 [references/upstreams.md](references/upstreams.md)。 - FRP/Caddy、PK01 shared Caddy managed block、public URL 暴露：读 [references/public-exposure.md](references/public-exposure.md)。 - master `~/.codex` 统一消费端配置：读 [references/local-codex-consumer.md](references/local-codex-consumer.md)。 - closeout 验收和最小 smoke：读 [references/validation.md](references/validation.md)。 -- 排障总入口：先读 [references/troubleshooting.md](references/troubleshooting.md)，再按失败层读取运行面、账号池或公网暴露专题。 +- 排障总入口： + - 先读 [references/troubleshooting.md](references/troubleshooting.md)，再按失败层读取运行面、账号池或公网暴露专题。 + - 客户可见错误溯源、错误率分析和配置调优必须继续读 [references/troubleshooting-accounts.md](references/troubleshooting-accounts.md)。 - 禁止事项和越界判断：读 [references/guardrails.md](references/guardrails.md)。 diff --git a/.agents/skills/unidesk-sub2api/agents/openai.yaml b/.agents/skills/unidesk-sub2api/agents/openai.yaml index 32e0717a..c35ac0b3 100644 --- a/.agents/skills/unidesk-sub2api/agents/openai.yaml +++ b/.agents/skills/unidesk-sub2api/agents/openai.yaml @@ -1,7 +1,7 @@ interface: display_name: "UniDesk Sub2API" - short_description: "Operate UniDesk Sub2API and Codex pool" - default_prompt: "Use $unidesk-sub2api to operate the UniDesk Sub2API Codex pool safely." + short_description: "运维 Sub2API、runtime 配置和 Codex 账号池" + default_prompt: "使用 $unidesk-sub2api 通过受控 CLI 运维 Sub2API，分析客户错误，并安全调整 Codex pool runtime 配置。" policy: allow_implicit_invocation: true diff --git a/.agents/skills/unidesk-sub2api/references/codex-pool.md b/.agents/skills/unidesk-sub2api/references/codex-pool.md index c9b06998..26811e27 100644 --- a/.agents/skills/unidesk-sub2api/references/codex-pool.md +++ b/.agents/skills/unidesk-sub2api/references/codex-pool.md @@ -4,11 +4,40 @@ - PK01 host-Docker 的 `sync --confirm` 通过 Sub2API admin API 对齐 group、YAML-managed accounts、统一消费 API key、capacity/loadFactor、WebSocket v2 标记和内置 `temp_unschedulable` 规则； - 统一消费 key 写入 YAML 声明的 `targets[PK01].hostDocker.envPath`； - PK01 路径不创建 k8s Secret、不部署 sentinel 资源，也不触发 `sub2api apply`、Docker compose、Caddy reload 或容器重启； - - `runtime list|get` 查询 pool group 内全部账号或指定账号的实际状态； - - `runtime errors` 按账号汇总 Sub2API 日志中的真实 `account_upstream_error` 状态码，并用 `openai.forward_failed` 提供脱敏错误摘要； - - `runtime apply|delete` 只修改显式 `--account` 指定账号的 `temp_unschedulable` 配置； + - `runtime list|get` 查询 pool group 内全部账号或指定账号的实际状态；`get` 同时披露脱敏 `modelMapping` 和完整临时不可调度规则； + - `runtime models --account` 通过 Sub2API 原生账号模型接口同时读取配置模型和上游实时模型，不把配置映射误当作真实上游能力； + - `runtime errors` 优先使用 Sub2API 原生 admin/ops 能力： + - `--all-groups` 按稳定 group ID 游标分页返回所有分组概要； + - `--group ` 返回指定分组的账号根因、策略效果和 request 索引； + - `--platform ` 可筛选平台；省略分组参数时继续分析 YAML 当前 Codex pool； + - 默认输出使用 Kubernetes 风格紧凑表格；机器读取必须显式使用 `--json`、`--full` 或 `--raw`； + - dashboard 请求量和客户错误率按分组精确统计； + - 账号被多个分组共享时，账号级 upstream-errors 和缺少 `group_id` 的 runtime 事件属于共享证据，禁止跨分组相加； + - dashboard overview 提供请求分母和错误率； + - account availability 提供可用、限流、过载和错误状态； + - concurrency 提供容量与排队； + - upstream errors 提供账号根因； + - 原生 system-log 索引提供事件时直接使用；索引未保留临时不可调度、failover 或 `forward_failed` 时，才从 target runtime 日志补这些事件，并显式披露 fallback 原因和数量； + - `policyEffectiveness` 从同一日志窗口汇总临时不可调度、切号、无候选和网关最终响应，并且只通过 `request_id` 关联切号结果； + - `requestDrilldown` 给出切号后成功、失败、结果缺失和选号失败的有限 request id，可继续交给 `trace` 下钻； + - 只有 `account_temp_unschedulable` 事件才算规则命中，不能仅凭错误状态码推断模板生效； + - `requestIdCoverage` 披露各类事件缺少 request id 的数量；缺失时不能把零关联解释为没有发生摘号后切号； + - `forwardFailureWithHttpSuccessRequestCount` 单列 HTTP 200 与 `forward_failed` 并存的请求，避免把流式响应已提交后的错误误判为完整成功； + - 网关最终错误占比只使用窗口内目标 API 路径的 `http request completed` 日志事件，必须同时披露窗口与 tail 边界，不能解释为完整业务错误率； + - `--full` 额外输出模型、小时、流式分布和有限详情样本； + - 当前并发是查询时快照，不代表错误发生瞬间并发；没有历史并发证据时，命令必须明确输出不可用，不能据此建议降低并发； + - 错误列表没有账号全部路由尝试数时，命令只报告错误数量，不伪造错误率； + - Ops 记录与 runtime 日志是独立证据，命令明确披露采样数量，不按日志位置强行关联； + - `runtime apply|delete` 支持 `temp-unschedulable` 和 `model-mapping` 两类 runtime 配置； + - 单账号必须显式使用 `--account`，精准批量必须显式使用 `--accounts `，不提供隐式 `all`； + - 批量操作先校验全部 selector 存在且账号类型可修改，再顺序调用 Sub2API admin API： + - 选择错误不会产生部分写入； + - 上游 admin API 在批次中途失败时仍需逐账号 `get` 对账； - 写操作不带 `--confirm` 时只返回 dry-run，带 `--confirm` 才通过 Sub2API admin API 写入； - - runtime 命令不输出 API key 或完整 credentials，默认返回紧凑 JSON，使用 `get`、`--full` 或 `--raw` 下钻； + - 批量默认输出账号 ID、名称、change、mode、mutation 和汇总计数；runtime 命令不输出 API key 或完整 credentials，使用 `get`、`--full` 或 `--raw` 下钻； + - 文档中的 runtime action 或参数被当前分支 CLI 拒绝时，按 CLI 集成漂移处理： + - 不退回手写 Sub2API 管理 API。 + - 先确认当前工作分支包含对应受控实现，或完成语义合入后再操作运行面。 - `sentinel-report`、`sentinel-probe`、`sentinel-image` 和部分 `trace` 能力仍以 k3s target 为主，需要这些能力时显式选择对应 k3s target。 k3s 账号池操作示例： @@ -19,10 +48,17 @@ bun scripts/cli.ts platform-infra sub2api codex-pool sync --target D601 --confir bun scripts/cli.ts platform-infra sub2api codex-pool validate --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool runtime list --target PK01 bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --target PK01 --account +bun scripts/cli.ts platform-infra sub2api codex-pool runtime models --target PK01 --account bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h +bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h --all-groups +bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h --group bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --account --template bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --account --template --confirm +bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --accounts --template +bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --accounts --template --confirm bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --target PK01 --account --kind temp-unschedulable +bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --account --kind model-mapping --model --upstream-model +bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --target PK01 --account --kind model-mapping --model bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --target D601 --confirm @@ -41,7 +77,14 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 - `pool.defaultTempUnschedulableTemplate`: YAML-managed 账号默认引用的临时不可调度模板。`sync --confirm` 只把模板渲染到 YAML-managed 账号。 - `profiles.entries[].tempUnschedulableTemplate`: 可选账号级模板引用，只用于明确偏离默认模板。 - YAML 未声明的账号标记为 `runtime-manual`；它可以显示 `matchingTemplate`，但模板差异不是 drift，也不能由 `sync` 自动改写。 -- 手工账号需要采用模板时，必须通过 `runtime apply --account ... --template ... --confirm` 显式写入；需要删除该类配置时使用 `runtime delete --account ... --kind temp-unschedulable --confirm`。 +- YAML 只保存可复用规则模板和 managed account 的模板引用。未在 YAML 预配置的手工账号 runtime 配置必须使用 CRUD，不得为了批量下发而把账号纳入 YAML managed profile。 +- 手工账号需要采用模板时： + - 写入使用显式 `runtime apply --account ...` 或 `--accounts ... --template ... --confirm`。 + - 删除使用相同显式 selector 的 `runtime delete --kind temp-unschedulable --confirm`。 +- 模型映射先用 `runtime models --account ...` 获取原生配置与实时上游能力： + - 只有实时模型列表或真实模型请求证明支持时才确认写入。 + - 禁止把名称相近的模型猜测为别名。 + - 禁止用错误映射掩盖 `model_not_found` 或无候选。 - runtime 模板与外部 `sentinel.*` 分开配置、互不驱动。内置规则负责请求路径即时冷却和 failover；哨兵负责 marker health、账号级隔离、恢复和 probe 退避。 - 外部 sentinel 的写入面只允许通过 Sub2API admin `schedulable` 接口冻结/恢复账号；不能写入、清理或间接清理 `temp_unschedulable_until` / `temp_unschedulable_reason`、rate-limit、overload、model-rate-limit 等 Sub2API 请求路径 runtime 状态，也不能调用 `recover-state` 作为恢复动作。看到 UI 里的“触发时间/解除时间/规则序号/匹配关键词”临时不可调度状态时，默认先归因到 Sub2API 内置 request-path temp-unschedulable，而不是 sentinel。 - YAML 只选择和配置 Codex 上游，不声明 `schedulable` 长期字段；`codex-pool sync --confirm` 不负责把既有账号恢复为 `schedulable=true`。既有账号的 `schedulable=false` 必须由哨兵先同步 Sub2API runtime 状态，再在 marker probe 命中后恢复。 @@ -60,7 +103,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 `sync --confirm` 会登录 Sub2API admin、创建/更新 group、创建/更新 YAML 中的 `unidesk-codex-*` accounts，并创建/复用统一 API key。k3s target 还会写入统一 API key Secret 并部署/更新哨兵资源；PK01 host-Docker target 只写 Sub2API runtime 和 host-Docker env 文件。host-Docker `validate` 也先使用 env 中的 Sub2API admin 凭据登录管理 API；如果 env 的统一 `API_KEY` 缺失或不匹配，它只通过管理 API 恢复或创建名为 YAML `pool.apiKeyName` 的统一消费 key 并写回 host-Docker env，不读取本地 profile 文件、不创建/更新账号、不 prune、不改变调度策略。`sync` 不把既有 managed account 直接恢复为 `schedulable=true`。恢复只由哨兵在读取 Sub2API runtime `schedulable=false` 后触发 recovery probe，并在 marker 命中时执行。`sync` 默认不删除 YAML 中缺席的 managed account。只有明确退役上游时才使用 `sync --confirm --prune-removed` 删除缺席且 `extra.unidesk_managed=true` 的 `unidesk-codex-*` account。对 `manualAccounts.protected`，`sync` 只执行 YAML 显式允许的窄同步；当前允许项是从目标 `egressProxy` 创建/更新 Sub2API internal proxy 记录并绑定 `proxy_id`，以及把受保护手动账号加入当前 `pool.groupName`。它仍不接管该账号凭据、status、schedulable、priority/capacity/loadFactor 或哨兵状态。若受保护手动账号在运行面缺失，`sync`/`validate` 会报告 manual account drift；不要自动创建、删除、接管或从 YAML 移除该账号。 -`trace --request-id ` 是只读 request 追溯报表，不触发 probe、不修改账号。默认输出请求开始/最终状态、failover、`account_select_failed`、窗口内 `account_temp_unschedulable`、admin schedulable 写入计数和当前账号快照；`reason=failover-attempted-no-candidate` 表示 Sub2API 已进入自动切号，但排除当前失败账号后没有可用候选。需要机器处理时使用 `--raw`，需要原始匹配行时加 `--show-lines`。 +`trace --request-id ` 是只读 request 追溯报表，不触发 probe、不修改账号。命令优先使用原生 `/admin/ops/requests` 获取请求结果，并使用 `/admin/ops/system-logs` 获取 request-id 索引事件；target runtime 日志只补原生索引未保留的邻近事件。报表显式输出 target、runtime mode、主数据源、上下文数据源和读取状态。默认输出请求开始/最终状态、failover、`account_select_failed`、`forward_failed`、窗口内 `account_temp_unschedulable`、admin schedulable 写入计数和当前账号快照。`forward_failed` 与 HTTP 200 并存且没有 failover 时，结果标记为 `degraded`，不能解释为完整流式成功；`reason=failover-attempted-no-candidate` 表示 Sub2API 已进入自动切号，但排除当前失败账号后没有可用候选。需要机器处理时使用 `--raw`，需要原始匹配行时加 `--show-lines`。 对已支持的 k3s target，`sync --confirm`、`validate` 和 sentinel 操作可能超过单次 SSH/runtime 短连接窗口；远端 job/poll 边界见 [sentinel.md](sentinel.md)。 diff --git a/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md b/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md index eeca3c0b..461a3359 100644 --- a/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md +++ b/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md @@ -1,5 +1,39 @@ ## 账号池排障 +### 客户可见错误与配置调优 + +1. 先用 `runtime errors --since ` 获取全账号同窗口证据： + - 以 Sub2API 原生 dashboard overview 的请求分母、客户错误率和上游错误率为主。 + - 以 account availability、concurrency 和 upstream errors 解释账号可用性、排队、状态码与根因。 + - 原生 system-log 索引缺少策略事件时，才使用 target runtime 日志补临时不可调度、failover、`account_select_failed` 和 `forward_failed`，并保留 fallback 披露。 +2. 从客户可见错误中选有限 request id 运行 `trace`，按请求链区分： + - 上游失败后成功切号。 + - 切号后候选耗尽。 + - 首次选号即无候选。 + - HTTP 200 已提交流式响应后出现 `forward_failed` 的 degraded 请求。 +3. 规则证据必须同时包含真实 HTTP 状态码与响应体关键词： + - 只有状态码、通用包装文案或推断出的原因不能扩展关键词。 + - 只有 `account_temp_unschedulable` 才证明规则命中；不能用相同状态码再次出现来推断规则失效。 + - 优先使用精确短语，避免把 `context window`、认证失败、模型不支持等不可互换错误混入通用不可用规则。 +4. 无候选或模型路由错误先运行 `runtime models`： + - 配置模型来自 `model_mapping`，实时模型来自上游原生同步，两者必须分开报告。 + - 没有账号真实支持目标模型时，保持池不声明该模型并快速失败，不映射到名称相近的其他模型。 + - 上游模型同步失败只能标记证据不足，不能据此补映射。 +5. 调优按所有权下发： + - 通用模板先修改 owning YAML，并保持冷却保守；默认从一分钟开始，没有新证据和明确授权不得超过三分钟。 + - YAML-managed 账号走 `sync`；runtime-manual 账号走显式 `runtime apply|delete`。 + - 多账号使用 `--accounts` 精准集合；例外账号从 selector 中显式排除，不使用隐式全量操作。 + - 先 dry-run 核对 change、before/after 和账号集合，再加 `--confirm`，最后逐账号 `get` 或批量 noop dry-run 对账。 +6. 不把当前并发快照当作错误时并发： + - 只有错误原文明确指向 concurrency，或存在错误时历史并发证据，才把降低上游并发作为有界对照实验。 + - 缺少直接证据时先处理主导上游根因。 +7. 下发后立即拉短窗口只做回归检查： + - 效果判定必须等待新的完整窗口。 + - 同时比较请求分母、客户错误率、上游错误率、规则命中、切号成功、候选耗尽和 degraded 流式请求。 +8. 版本回退必须有源码差异或版本对照证据证明当前版本引入回归： + - 旧版具有同一选择或错误传播逻辑时，不用回退替代配置治理。 + - 新版包含相关 failover 修复时，优先保留新版并治理真实根因。 + - Codex pool 哨兵、账号冻结/恢复、marker-only 判断或 probe 周期看不清：第一步跑 `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report`。这个报表是主观察面；只有报表缺字段或需要底层证据时，才继续看 `--raw`、CronJob log、state ConfigMap 或 Sub2API 管理 UI。若看到“临时不可调度状态”且包含规则序号/匹配关键词，检查 Sub2API `account_temp_unschedulable` 日志和账号 `temp_unschedulable_*` 字段；sentinel 只解释 `schedulable=false` 的 active quarantine，不解释这类内置临时冷却。 - 只加强监控、不让哨兵自动冻结账号时，把 YAML `sentinel.actions.enabled=false` 后 `codex-pool sync --confirm`。此时 marker probe 和 gateway failure monitor 仍记录 `would-freeze` / observe-only 证据，但不会通过 Sub2API admin 写 `schedulable=false`；`/responses/compact` 的 `codex.remote_compact.failed` 和 compact 上游 5xx failover 只作为 `gateway-compact-*` 观察事件记录，不作为哨兵自动切换触发器。 - 单个 request id 报 502/503/中断/没有自动切号：第一步跑 `bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id `。先看 `outcome`、`reason`、`FAILOVER`、`SELECT-FAILED`、`ACCOUNT SIGNALS` 和 `WINDOW STATS`；只有 trace 报表缺字段或需要审计原始日志时，才加 `--show-lines` 或 `--raw`。若 `reason=failover-attempted-no-candidate`，说明切号动作已发生，但 scheduler 在排除失败账号后没有可用候选；继续用 `sentinel-report` 和 `validate --full` 区分 sentinel quarantine、request-path temp-unschedulable、账号 status 或容量耗尽。 diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 31a138f0..9b9702ae 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -12,22 +12,30 @@ runtime: templates: - id: codex-upstream-failover kind: temp-unschedulable - description: Reusable Sub2API request-path cooling and failover policy for Codex-compatible API-key accounts. + description: Codex 兼容 API-key 账号的请求路径短时冷却与换号策略。 spec: enabled: true rules: + - statusCode: 429 + keywords: [concurrency limit exceeded] + durationMinutes: 1 + description: 明确命中上游并发限制时短时冷却当前账号。 + - statusCode: 500 + keywords: [failed to validate api key, do request failed] + durationMinutes: 1 + description: 上游 API-key 查询依赖或请求传输内部失败时短时冷却当前账号。 - statusCode: 502 - keywords: [upstream service temporarily unavailable, upstream request failed, context window, stream usage incomplete, missing terminal event] + keywords: [upstream service temporarily unavailable, upstream request failed, service temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 - description: Observed 502 upstream availability, context-window, and incomplete-stream failures briefly cool down the selected account. + description: 明确命中 502 上游不可用、过载或并发限制时短时冷却当前账号。 - statusCode: 503 - keywords: [upstream service temporarily unavailable, upstream request failed, temporarily unavailable] + keywords: [upstream service temporarily unavailable, upstream request failed, service temporarily unavailable, temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 - description: Observed 503 temporary upstream availability failures briefly cool down the selected account. + description: 明确命中 503 上游暂时不可用、过载或并发限制时短时冷却当前账号。 - statusCode: 524 - keywords: [upstream request failed, missing_required_parameter, encrypted_content] + keywords: [upstream request failed, service temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 - description: Observed 524 upstream request and encrypted-content failures briefly cool down the selected account. + description: 明确命中 524 上游不可用、过载或并发限制时短时冷却当前账号。 pool: groupName: unidesk-codex-pool diff --git a/docs/MDTODO/details/observability-trace-reliability/R4.4.2_Task_Report.md b/docs/MDTODO/details/observability-trace-reliability/R4.4.2_Task_Report.md new file mode 100644 index 00000000..1b6248bf --- /dev/null +++ b/docs/MDTODO/details/observability-trace-reliability/R4.4.2_Task_Report.md @@ -0,0 +1,21 @@ +# R4.4.2 任务报告 + +## 当前状态 + +任务仍在进行中。UniDesk #1959 / PR #1961 已解决 runtime GitOps 大段脚本内联导致的 ARG_MAX 问题，后续 PR 已修复 ConfigMap ownership、YAML 解析和 terminal marker 组合；这些修复保留。 + +## 最新架构纠偏 + +此前把 direct/live/refresh/projector/outbox/realtime 六个架构布尔值同时投影到 YAML、renderer、Deployment env、应用 parser 和 CI verify，已经造成合法自动链被错误阻塞，并使缺少单个 env 的新 cloud-api Pod CrashLoop。该模型已废弃： + +- 数据 authority 与事件路径固定为 agentrun.event.v1 → transactional projector → PostgreSQL facts/outbox → hwlab.event.v1 → realtime/replay SSE。 +- 架构不再提供 env/YAML 开关，旧 HWLAB #2536 已关闭。 +- 产品功能使用仓库内 config/feature-config.schema.json，一个功能只对应一个规范变量。 +- schema 缺失、非法、不匹配或重复只产生 typed warning；不得写回配置、关闭功能、切换路径或阻塞滚动上线。 + +## 当前执行 + +- UniDesk #1981 / MDTODO R6：删除 CI/CD 六 env 硬门禁并实现只读 warning schema 校验；Artificer task qt_7d73811bd7194ea186c904645da8be6c。 +- HWLAB #2538 / MDTODO R4.4.2.10：删除运行时架构 env 和旧分支，固定纯 Kafka transactional 投影；Artificer task qt_37898861dcea44219115c91075af340f。 + +任务将在 PR 审核、正常 merge 自动交付、NC01/v03 新 Pod Ready、Argo Synced/Healthy、Kafka 健康指标归零及新 hi 实时/回放/canonical duration 验收后收口。 diff --git a/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R6_Task_Report.md b/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R6_Task_Report.md new file mode 100644 index 00000000..9519de76 --- /dev/null +++ b/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R6_Task_Report.md @@ -0,0 +1,22 @@ +# R6 任务报告 + +## 结论 + +- 产品功能 schema 固定为 `config/feature-config.schema.json`，使用 JSON Schema Draft 2020-12 与锁定的 `ajv-dist@8.17.1` 结构化 validator。 +- 每个顶层 property 通过唯一 `x-unidesk-feature` 建立一对一配置关系；CI runner env 不作为产品配置 authority。 +- schema、输入、重复、值不匹配、validator 与 OTel 异常均只产生 typed warning，不写回配置、不关闭功能、不切换架构，也不阻断 artifact、GitOps、Argo、rollout 与 health。 +- PR #1987 完成 schema/validator/OTel/`status|history|debug-step` 投影和旧 capability env 门禁删除；PR #1989 修复 source artifact 缺少 Ajv bundle 的首断点。 + +## 验证 + +- 定向测试首轮 53/53；主代理 shell 状态纠偏后回归 49/49。 +- merge commit `92b19ce6c8b55b1059506d6b4527388d1f671828` 自动创建 `unidesk-host-v2xn8`，44 秒 Succeeded。 +- GitOps `fa4d36a288ac2943091ffb92e0a4d5a0e93e5ff5`；Argo `Synced/Healthy` 且 revision relation=exact。 +- registry/runtime digest 同为 `sha256:a10a22b94976eb65a664b0b3b0c838a5720d39c037229b325c6fe0083578f355`，1/1 ready，health=200。 +- 全程未人工创建 PipelineRun、mirror sync、Argo refresh 或 patch 运行面。 + +## 工件 + +- Issue: https://github.com/pikasTech/unidesk/issues/1981 +- PR: https://github.com/pikasTech/unidesk/pull/1987 +- Follow-up PR: https://github.com/pikasTech/unidesk/pull/1989 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R1_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R1_Task_Report.md new file mode 100644 index 00000000..2cbe3888 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R1_Task_Report.md @@ -0,0 +1,37 @@ +# R1 任务报告 + +## 目标 + +把 Sub2API 上游故障调查从零散日志读取收敛为可复用的受控 CLI 能力，并用实际错误证据形成保守、可逆的账号级 runtime 调度缓解。 + +## 工具改进 + +- `runtime list|get` 补充 `proxyId`、并发上限、当前并发、`loadFactor` 和优先级，能够直接核对代理与调度事实。 +- `runtime errors` 综合 `account_upstream_error`、`openai.forward_failed` 和 Sub2API Ops 上游错误记录，输出全池主因排名、账号主因、状态码、消息、模型、小时和流式分布；详细样本通过 `--full` 渐进披露。 +- 错误分析显式区分 Ops 样本与 runtime 日志，不按时间或位置强行关联；缺少全部请求尝试分母时不计算错误率，缺少错误发生时并发历史时不把当前并发快照作为降并发依据。 +- `runtime apply|delete` 仅修改显式指定账号的临时不可调度配置；无 `--confirm` 时为 dry-run，有 `--confirm` 时才写入，并回读模板匹配、代理、并发、状态和调度状态。 + +## 全池结论 + +- 最近观测窗口内主要错误是仅状态码上游失败、通用上游失败、过载和上游 API-key 校验失败；明确并发错误只占较小部分，降低全池并发量不应作为首要修复。 +- 账号 27 的集中 HTTP 500 证据为 `Failed to validate API key`，指向上游网关 key 查询或依赖失败；账号 32 才有明确并发限制证据。 +- context window、缺少 terminal event 和 `encrypted_content` 属于请求语义或流式续链问题，不应通过临时摘号模板泛化掩盖。 +- Sub2API 仅能在响应尚未提交时通过临时不可调度触发换号；SSE 已向客户端写出后再缺少终止事件，不能透明切换账号。 + +## 配置与运行面应用 + +- owning YAML 的 `codex-upstream-failover` 模板收敛为 5 条实际观测规则：`429`、`500`、`502`、`503`、`524`，每条冷却 1 分钟。 +- `500` 仅匹配实际证据中的 `failed to validate api key` 和 `do request failed`；可用性、过载和并发关键词按实际状态码分别配置。 +- 通过 runtime CRUD 将模板显式应用到 9 个 API-key 手工账号；OAuth 账号 `lyon9801` 保持 `enabled=false`、零规则。 +- 未改变账号凭据、代理、并发上限、优先级、长期 `status` 或 `schedulable`；原先已禁用或错误的账号保持原状。 + +## 验证 + +- 全池 runtime 回读确认 9 个 API-key 账号均为 5 条规则并匹配模板，OAuth 例外未变化。 +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts` 通过。 +- Bun 内置 YAML 解析确认规则状态码为 `429,500,502,503,524`，持续时间均为 1 分钟。 +- `git diff --check` 通过。 + +## 后续边界 + +本次只能证明配置正确应用，不能立即证明错误率下降。后续应在独立的新时间窗口复核错误记录、切号证据和最终客户响应，并继续保持错误数量、请求分母和流式提交阶段的语义区分。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.1_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.1_Task_Report.md new file mode 100644 index 00000000..e494a8bc --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.1_Task_Report.md @@ -0,0 +1,55 @@ +# R2.1 任务报告 + +## 目标 + +增强 UniDesk Sub2API CLI，优先使用 Sub2API 原生 admin/ops 运维能力诊断全池错误、账号状态、并发和单请求链路，运行日志只补原生索引未提供的策略事件。 + +## 官方能力依据 + +对照 `Wei-Shaw/sub2api` 官方仓库提交 `5d1c577cb2c735ca1f1d57533dff1302f6998d91`，确认可复用接口： + +- `/api/v1/admin/ops/dashboard/overview`：请求总数、客户可见错误、错误率、上游错误率、健康分。 +- `/api/v1/admin/ops/upstream-errors`：账号级上游错误、状态码、request id 和详情。 +- `/api/v1/admin/ops/account-availability`：账号可用、限流、过载和错误状态。 +- `/api/v1/admin/ops/concurrency`：平台、组和账号并发占用、容量与排队。 +- `/api/v1/admin/ops/requests`：成功与错误请求的统一 request-id 视图。 +- `/api/v1/admin/ops/system-logs` 及 `/health`：索引日志和 sink 健康状态。 + +## CLI 改进 + +- `runtime errors` 增加 `nativeOps`，使用 overview 的完整请求分母计算客户可见错误率，避免把错误记录数当错误率。 +- 增加原生账号可用性、组并发、排队和 system-log sink 丢弃/写入失败状态。 +- Ops 上游错误改为所有选中账号主动查询，不再要求容器日志先出现对应事件。 +- `policyEffectiveness` 关联临时不可调度、failover、选号失败、`forward_failed` 和最终结果；仅按 request id 关联并披露缺失覆盖率。 +- 原生 system-log 索引没有策略事件而 overview 存在上游错误时，自动回退 target runtime 日志补事件，并披露 fallback 原因、数量和原生索引计数。 +- 全池默认只输出主因排名，账号详情通过 `--account` 下钻；全池和单账号默认输出均保持在 CLI 预算内，`--full` 保留完整分布。 +- `trace` 改为 `/ops/requests` 与 `/ops/system-logs` 优先，target runtime 日志只补邻近上下文。 +- 修复 PK01 trace 固定读取 k3s Deployment、Docker stderr 业务日志遗漏和原生事件倒序问题。 +- `forward_failed` 与 HTTP 200 并存且没有 failover 时，trace 标记为 `degraded`，不再误报普通成功。 +- 修正 `No available channel for model` 根因分类和所有 Ops 错误被描述为 HTTP 500 的误导文案。 + +## 现场验证 + +- 10 分钟原生 overview 样本曾观测 165 个请求、2 个客户可见错误，错误率 1.21%，上游错误率 3.03%；后续窗口为 189 个请求、1 个客户可见错误，错误率 0.53%，上游错误率 2.65%。这些是滚动窗口快照，不代表长期趋势。 +- 原生 concurrency 显示组容量 218，现场占用约 9 至 10，无排队；全池错误不能归因于 Sub2API 组并发耗尽。 +- 原生 availability 显示账号 29、30 因 `403 Insufficient account balance` 处于 error；账号 27 在短时冷却窗口可能暂不可用。 +- 策略事件样本显示临时不可调度和 failover 均实际发生；部分切号成功，仍存在切号后失败、结果缺失和 `forward_failed + HTTP 200`。 +- request `03a94444-ff26-4038-b1b9-67e249ce7f59`：账号 33 返回 524 后切到账号 31，账号 31 返回 502，随后 `account-select-failed/context canceled`，最终客户状态 502。 +- request `55d32c6a-09ae-46ac-9a34-3f63934e7f62`：账号 31 出现并发错误但 HTTP 200，没有 failover，trace 正确标记 `degraded`。 +- 账号 31 的 10 分钟 Ops 记录同时包含 429 rate-limit 和明确 concurrency 文案；当前并发快照不能替代错误发生时并发，后续只能通过受控对比判断是否需要调整账号并发。 +- 原生 system-log sink 健康，无 dropped/write failed；但策略事件查询为零，当前需要 runtime 事件 fallback。 + +## 验证命令 + +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts` +- `bun --check scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts` +- `bun --check scripts/src/platform-infra-sub2api-codex/render.ts` +- `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 10m` +- `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 10m --account 31` +- `bun scripts/cli.ts platform-infra sub2api codex-pool trace --target PK01 --request-id 03a94444-ff26-4038-b1b9-67e249ce7f59` +- `git diff --check` + +## 后续 + +- R2.2 持续观察新窗口，区分客户可见错误率、上游错误率、成功 failover 和流式 degraded。 +- R2.3 调查原生 system-log 索引为何未保留策略事件，并优先推动 Sub2API 原生能力覆盖，减少 runtime 日志 fallback。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.1_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.1_Task_Report.md new file mode 100644 index 00000000..ba555db9 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.1_Task_Report.md @@ -0,0 +1,107 @@ +# R2.2.1 客户可见错误溯源与调优建议 + +## 范围与数据源 + +- 运行目标：PK01 host-Docker，Sub2API `0.1.153`。 +- 原生 API：`dashboard/overview`、`upstream-errors`、`account-availability`、`concurrency`、`requests`、`system-logs/health`。 +- 原生 `system-logs` 未索引策略事件，按既定 CLI 规则回退读取 runtime 日志。 +- 观察窗口：连续 20 分钟快照与 60 分钟汇总。 +- 本轮只分析，没有修改 YAML、runtime 配置或 Sub2API 代码。 + +## 客户错误率 + +- 20 分钟窗口：335 次请求，4 次客户错误，客户错误率 1.19%，上游错误率 2.99%。 +- 60 分钟窗口：907 次请求，8 次客户错误，客户错误率 0.88%，上游错误率 3.64%。 +- 60 分钟内发生 7 次 `account_select_failed`，几乎覆盖全部客户可见错误。 +- 组并发快照为 9/218，负载 4.13%，排队 0；全池并发不是主因。 +- system-log sink 的 queue、drop 和 write failure 均为 0；日志落库不是业务错误来源。 + +## 请求级根因 + +### 模型候选为零 + +- 三个 `gpt-5.4-nano` 流式请求分别在 5ms、10ms、28ms 内未选中账号并返回 404。 +- 请求 ID：`21e5d449-3b0d-4a70-a798-1dc0af0ba110`、`55f7eb2a-b469-4d90-9ec9-3d20d278aef7`、`e90dfd63-22c1-419b-a479-53f860d4151a`。 +- 官方源码会在组内存在账号但没有任何 `model_mapping` 支持请求模型时返回 `404 model_not_found`。 +- 这不是瞬时容量问题，重试、临时不可调度和提高切号次数均无效。 + +### 多账号连续失败后候选耗尽 + +- `8150234d-eee2-4a08-a43e-e47a22274609`：`gpt-5.4`，account 28 返回 503、account 27 返回 429、account 31 返回 429，约 6.1 秒后无候选，最终 429。 +- `bf9aea0a-869a-4ee0-a99c-3b0ceb553822`：`gpt-5.6-luna`，account 28 返回 503、account 27 返回 429，约 67 秒后上下文取消且无候选，最终 429。 +- `03a94444-ff26-4038-b1b9-67e249ce7f59`：`gpt-5.6-sol`，account 33 返回 524、account 31 返回 502，约 180 秒后上下文取消且无候选，最终 502。 +- `5d8f05ff-9cbc-4040-b6eb-4b262ab85f39`：`gpt-5.6-sol`，account 33 返回 524，约 126 秒后上下文取消且无候选，最终 502。 +- `max_account_switches=10` 只是上限。官方实现会排除已失败账号并重新选号；剩余账号不满足模型或运行状态时，会在只切 1 至 3 次后结束。 + +### 流式提交后错误 + +- 成功样本证明自动 failover 正常工作：account 27 的 502 可切到 account 33，account 28 的 503 可切到 account 31。 +- 存在 `forward_failed + HTTP 200`，例如 account 31 返回 `Concurrency limit exceeded`，但下游已经提交流式响应。 +- 官方 `v0.1.153` 在检测到已写入语义响应后明确禁止换号，以免拼接两个上游流。 +- 此类错误不能靠临时不可调度透明恢复，只能减少上游发生率，或由客户端在流中断后重试或恢复。 + +## 上游账号主因 + +- 60 分钟上游记录共 72 条：rate-limit 24、overload 15、concurrency 12、generic upstream failure 11、status-only 6、model-route-unavailable 4。 +- account 28：10 次 overload，当前高频触发 503 临时不可调度。 +- account 27：rate-limit 为主，同时有 502 generic failure；现有规则只覆盖 502，不覆盖真实 429 文本。 +- account 31：13 次 rate-limit 为主，另有 concurrency 和一次 `GROUP_DISABLED`；模板未匹配 `Upstream rate limit exceeded`。 +- account 19：7 次 504，其中 2 次明确 concurrency、1 次 upstream unavailable、4 次只有状态；另有 3 次 `No available channel for model`。 +- account 32：2 次 429 rate-limit、1 次 502 concurrency。 +- account 29、30：余额不足导致 403，已处于 `status=error`、`schedulable=false`，不会继续被调度，但减少了候选容量。 + +## 官方版本判断 + +- PK01 运行 `v0.1.153`。 +- `v0.1.152..v0.1.153` 的相关变更是 WebSocket ingress 生命周期限制，不改变当前 HTTP 模型选号、failover 或临时不可调度匹配。 +- `v0.1.149` 已包含无账号支持模型时返回 404 的实现，因此回退不会消除 `gpt-5.4-nano` 404。 +- `v0.1.153` 已包含 compact 心跳不阻断 failover 的官方 #3887 修复。没有证据表明回退能降低当前错误率，不建议回退。 + +## 调优建议 + +### P0：修正模型目录与映射 + +- 查明 `gpt-5.4-nano` 是客户端误配、错误广告模型还是预期别名。 +- 若没有真实上游支持，从客户可见模型目录和调用配置中移除。 +- 若业务确实需要，只映射到已经实测支持的真实模型，不能为消除 404 添加虚假映射。 + +### P1：扩展保守临时不可调度模板 + +- 在 429 规则增加真实关键词 `upstream rate limit exceeded`，保持 1 分钟冷却。 +- 新增 504 规则时只加入实测关键词 `concurrency limit exceeded` 和 `upstream service temporarily unavailable`，保持 1 分钟冷却。 +- 不增加任意 504 规则；4 次 504 只有状态码，不能推断真实原因。 +- 不把 `No available channel for model` 放入通用账号级模板；这是模型级缺口，账号级摘除会误伤其他模型。 +- `lyon9801` 继续保持例外，不绑定临时不可调度规则。 + +### P1：账号级并发对照调优 + +- 不降低组级并发。 +- account 31 有最强直接并发证据，可先做单账号对照，例如从 30 降到 20，观察 30 至 60 分钟的错误和吞吐。 +- account 19、27、32 也有信号，但没有错误时刻并发历史；不要一次性全降。 +- 手工账号属于 runtime 管理面，后续通过 runtime CRUD 调整，不纳入 YAML managed profile 自动同步。 + +### P1：降低慢失败对客户时间预算的吞噬 + +- 524 请求分别在约 126 秒和 180 秒后穿透，第一个慢上游已消耗大部分客户超时预算。 +- 官方默认 `gateway.response_header_timeout=600s`，明显长于当前客户或边缘上下文。 +- 建议先确认运行实际值，再受控收紧 OpenAI 响应头超时，为一次 failover 预留预算；不能直接照抄固定值。 +- 对照指标应包含首 token 延迟、超时型上游错误、failover 成功率和客户错误率。 + +### P2：恢复候选容量与调度偏好 + +- account 29、30 只有补足余额并通过真实请求验证后才能恢复，不能直接改为 schedulable。 +- account 28、27、31 当前优先级高且错误集中，可在确认模型覆盖后降低调度偏好，让更稳定账号先承担请求。 +- Ops system-log 未索引策略事件，60 分钟有 144 条事件依赖 runtime fallback；这不造成业务错误，但限制长期归因，由 R2.3 跟踪。 + +## 不建议的措施 + +- 不建议回退到 `v0.1.149`。 +- 不建议提高 `max_account_switches`。 +- 不建议降低全池并发。 +- 不建议启用 `pool_mode` 做同账号重试。 +- 不建议对所有 4xx/5xx 做无关键词临时不可调度。 + +## 后续验收 + +- 按 20 分钟和 60 分钟双窗口继续观察客户错误率、upstream error rate、`account_select_failed`、模型 404、failover 成功率、`forward_failed + 200` 和每账号根因。 +- 每次只改变一个变量并保留前后窗口；R2.2 继续保持进行中。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.2_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.2_Task_Report.md new file mode 100644 index 00000000..9d7afedc --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.2_Task_Report.md @@ -0,0 +1,64 @@ +# R2.2.2 任务报告 + +## 目标 + +执行三项客户错误调优： + +1. 处理 `gpt-5.4-nano` 无候选。 +2. 为 429 临时不可调度规则加入真实上游关键词 `upstream rate limit exceeded`。 +3. 增加仅匹配 `concurrency limit exceeded` 与 `upstream service temporarily unavailable` 的 504 规则。 + +同时按用户追加要求增强精准批量 runtime API。 + +## 模型能力结论 + +通过 Sub2API 原生账号接口 `GET /api/v1/admin/accounts/:id/models` 和 `POST /api/v1/admin/accounts/:id/models/sync-upstream` 检查账号 15、19、27-34： + +- 所有账号当前 `model_mapping` 均不包含 `gpt-5.4-nano`。 +- 可实时同步的账号 19、27、28、32、33、34 的上游模型列表均不包含该模型。 +- 账号 29、30、31 的上游模型同步返回 HTTP 403，不能据此声明支持。 +- `lyon9801` 为 OAuth 账号，Sub2API 原生同步接口明确不支持该类型，当前映射也不包含该模型。 + +因此没有创建虚假的 `nano -> mini` 或同名映射。当前池的 `/v1/models` 不宣称支持 `gpt-5.4-nano`，直接请求继续快速返回无候选 404。后续只有真实上游模型列表或实测请求证明支持时，才通过显式 model-mapping CRUD 加入。 + +## 配置调优 + +YAML 模板 `codex-upstream-failover` 保持一分钟冷却，并累计为六条状态规则： + +- 429：`concurrency limit exceeded`、`upstream rate limit exceeded`。 +- 500：`failed to validate api key`、`do request failed`。 +- 502、503、524：保留前序已验证的上游不可用、过载和并发关键词。 +- 504：仅 `concurrency limit exceeded`、`upstream service temporarily unavailable`。 + +通过受控 runtime CLI 对账号 19、27-34 dry-run 后确认下发。逐账号回读均为 `ruleCount=6`、状态码 `429/500/502/503/504/524`，且匹配模板。账号 15 `lyon9801` 保持 `ruleCount=0`，没有应用临时不可调度。 + +## CLI 增强 + +新增： + +- `runtime models --account `：同时显示脱敏 `model_mapping`、配置模型和原生上游实时模型。 +- `runtime apply|delete --kind model-mapping`：显式单账号模型映射 CRUD，默认 dry-run。 +- `runtime apply|delete --accounts `：一次受控作业内按显式账号集合顺序执行，拒绝隐式 all、重复账号和非 API-key 账号。 +- 批量默认输出压缩为账号 ID、名称、change、mode、mutation 和汇总计数；`--full/--raw` 保留完整计划。 + +批量验收命令精确选择账号 `19,27,28,29,30,31,32,33,34`，结果为 `requestedCount=9`、`succeededCount=9`、`mutatedCount=0`，全部 noop，且未包含账号 15。 + +## 即时观察 + +调优后拉取 20 分钟 runtime 错误窗口： + +- 匹配 12 个日志事件。 +- 结构化上游状态为 502 一次、524 六次。 +- 账号 27 仍出现四次并发限制文本，但该窗口跨越配置下发时刻，不能用于评价新规则最终效果。 +- 账号 33 有一次 `gpt-5.6-sol` 与 ChatGPT Codex 账号不兼容的 400，属于独立模型路由问题。 +- 未观察到配置写入造成的新类型异常。 + +父任务 R2.2 继续用后续完整窗口观察客户可见错误率、切号和冷却命中效果。 + +## 验证 + +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts`：通过。 +- `git diff --check`：通过。 +- 九账号 runtime dry-run、confirm、逐账号 get：通过。 +- 精准批量 dry-run：通过。 +- 20 分钟 runtime errors：通过。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.3_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.3_Task_Report.md new file mode 100644 index 00000000..e0c9dc3c --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.3_Task_Report.md @@ -0,0 +1,53 @@ +# R2.2.3 任务报告 + +## 目标 + +把本轮形成的 Sub2API runtime 控制、原生数据诊断、客户可见错误溯源和配置调优方法沉淀到 `unidesk-sub2api` skill，避免后续代理重新试探入口、证据口径和下发边界。 + +## 沉淀内容 + +### Skill 触发与路由 + +- 扩展 skill description，使 runtime CRUD、精准批量、临时不可调度、上游错误率、客户可见错误、failover、模型映射和可用模型探测均能触发本 skill。 +- `SKILL.md` 将 runtime 控制路由到 `references/codex-pool.md`。 +- 客户错误溯源、错误率分析和配置调优路由到 `references/troubleshooting-accounts.md`。 +- 同步更新 `agents/openai.yaml` 的简述和默认提示。 + +### Runtime 控制合同 + +`references/codex-pool.md` 固化： + +- `runtime list|get|models|errors|apply|delete` 的职责。 +- `get` 的脱敏 `modelMapping` 与临时不可调度规则可见性。 +- `models` 区分配置模型和上游实时模型。 +- `temp-unschedulable` 与 `model-mapping` 两类 CRUD。 +- 单账号 `--account` 和显式精准批量 `--accounts`，禁止隐式 `all`。 +- 批量选择预检、顺序写入、中途 admin API 失败后的逐账号对账要求。 +- 默认 dry-run、`--confirm` 写入、紧凑批量输出和 Secret/credentials 脱敏。 +- YAML 只保存通用模板和 managed 引用，runtime-manual 账号继续使用 CRUD，不为批量操作强行纳入 YAML。 +- 文档命令与当前分支 CLI 不一致时按集成漂移处理，禁止退回手写管理 API。 + +### 调优分析流程 + +`references/troubleshooting-accounts.md` 固化八步流程： + +1. 用原生 dashboard、availability、concurrency、upstream errors 获取全账号同窗口事实。 +2. 用有限 request id 和 `trace` 区分切号成功、候选耗尽、首次无候选和流式 degraded。 +3. 临时不可调度规则必须同时有真实状态码与响应关键词证据。 +4. 无候选先用原生模型接口核实能力，不猜测近似模型映射。 +5. 按 YAML-managed 与 runtime-manual 所有权分别下发，默认一分钟冷却且无新证据不超过三分钟。 +6. 不用当前并发快照证明错误时并发，降低并发只做有直接证据的有界对照。 +7. 短窗口只做回归，效果等待新的完整窗口并综合请求分母、错误率、规则命中和切号结果。 +8. 版本回退必须有源码差异或版本对照证据。 + +## 实现边界 + +精准批量和 `runtime models` 已在 `/root/unidesk/.worktree/sub2api-error-tuning` 完成真实 PK01 验证。主工作区当前 runtime CLI 尚未语义吸收这部分实现，且同文件存在并行诊断增强修改，因此本任务没有覆盖主工作区 runtime 源码。skill 已加入 CLI 集成漂移防护，后续必须先完成语义合入，不能手写 Sub2API admin API 绕过。 + +## 验证 + +- `quick_validate.py .agents/skills/unidesk-sub2api`：通过。 +- `git diff --check`：通过。 +- skill 主文件 46 行，三个核心文件合计 207 行，保持渐进披露。 +- `agents/openai.yaml` 的 `default_prompt` 显式引用 `$unidesk-sub2api`。 +- 未写入账号 ID、供应商名称、当前错误率或其他一次性运行数据。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.4_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.4_Task_Report.md new file mode 100644 index 00000000..a523f848 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.4_Task_Report.md @@ -0,0 +1,104 @@ +# R2.2.4 所有分组客户错误与 CLI 渐进披露报告 + +## 交付范围 + +- `runtime errors --all-groups`：按稳定 group ID 游标分页，默认返回所有分组概要。 +- `runtime errors --group `：下钻单个分组的原生 Ops 指标、账号根因、策略效果和 request ID 索引。 +- `--platform `：可选平台过滤。 +- `--page-token `：固定分页游标，不使用手工 `limit` 规避输出问题。 +- `runtime list|get|errors|apply|delete`：默认统一为 Kubernetes 风格紧凑表格。 +- `--json`、`--full`、`--raw`：仅在显式请求时返回机器结构或完整披露。 + +## 运行面证据 + +- 目标：PK01 host-Docker。 +- 实际镜像：`docker.1panel.live/weishaw/sub2api:0.1.153`，容器 healthy。 +- 数据源：Sub2API 原生 dashboard overview、account availability、concurrency、upstream-errors、requests 和 system-logs。 +- 原生 system-log 没有保留全部策略事件时，CLI 显式回退到 runtime 日志。 +- system-log sink 的 queue、drop 和 write failure 均为 0。 + +## 所有分组概要 + +同一 60 分钟查询页共 3 个分组： + +- 全局：757 次请求，8 次客户错误，客户错误率 1.06%，34 次上游错误，上游错误率 4.49%。 +- `unidesk-codex-pool`，group 2：277 次请求，6 次客户错误，错误率 2.17%；21 次上游错误，上游错误率 7.58%。 +- `自用`，group 3：480 次请求，2 次客户错误，错误率 0.42%；13 次上游错误，上游错误率 2.71%。 +- `grok`，group 5：窗口内无请求和错误。 +- group 2 承担 36.6% 请求，却产生 75% 客户错误，是当前主要客户错误来源。 + +查询时两个活跃分组均为 3/218 并发、队列 0。降低全局或分组并发不能解释当前错误，不应作为主修复。 + +## 账号和共享证据边界 + +- group 2、3 共享大部分 OpenAI 账号。 +- 原生 dashboard 请求量和客户错误率按 group 精确统计，可以跨组汇总。 +- upstream-errors 是账号级数据；共享账号的根因记录会出现在多个 group 下钻中，禁止跨组相加。 +- 缺少 `group_id` 的临时摘号事件只能按账号 membership 和 request ID 关联，CLI 已标注为共享证据。 +- 原生聚合显示每个活跃组 9 个账号、6 个可用、3 个错误账号。 +- 错误账号为 29、30 的余额不足 403，以及 31 的 API Key 所属分组已删除 403。 + +账号共享窗口内的 34 条上游根因记录为： + +- overload：18。 +- status-only-upstream-failure：9。 +- generic-upstream-failure：7。 + +## 客户可见错误链 + +### 长耗时失败后 context 已取消 + +- `cf6b6257-9640-414f-918d-c55180fb6a8d`：`gpt-5.6-luna`，账号 27 返回 502，约 41.6 秒后进入 failover，下一次选号立即报 `context canceled`，最终 502。 +- `4ee54daa-f41c-4f50-bd72-11080c38e299`：`gpt-5.6-sol` 流式请求，账号 27 返回 502，约 730.8 秒后进入 failover，下一次选号立即报 `context canceled`，最终 502。 +- 两条请求均已触发切号；失败点是第一个上游耗尽了客户请求时间预算，不是 `max_account_switches` 太小，也不是池容量不足。 + +### HTTP 200 后流式失败 + +- `46c1374a-c7f1-49f8-877d-d7ffb216976c`：账号 28 返回 503 后切到账号 33，网关最终 access status 为 200。 +- 随后出现 `openai.forward_failed`，原文指向 `input[2].encrypted_content` 的 ObjectParam/required-parameter 错误。 +- CLI 现判定 `outcome=degraded`、`reason=forward-failed-http-success`，不再把 HTTP 200 误报为完整成功。 +- 流式语义已经提交后不能安全拼接另一个上游响应，透明换号只适用于提交语义响应之前。 + +## 临时不可调度和 failover 效果 + +- group 3 下钻观察到 20 次临时不可调度事件、13 个 failover 请求。 +- 其中 2 个明确成功、1 个明确失败、10 个结果在有限日志索引中缺失。 +- 另有 1 个 `forward_failed + HTTP 200` degraded 请求。 +- 规则能减少立即重复命中坏账号，但无法恢复已经耗尽 context 的请求，也无法恢复已经提交语义流后的协议错误。 + +## 官方仓库与版本判断 + +- 官方 v0.1.153 发布说明只涉及调度缓存异常时间、WebSocket 生命周期等修复，没有声明改变当前 HTTP Responses 的 failover 时间预算或流式提交后切号规则。 +- 官方 issue #2067 与合并 PR #2068 证明旧版曾存在 Responses 多轮 reasoning item 回放导致 404/502 的协议转换缺陷；修复已在 2026-04-29 合入主线。 +- 当前 `encrypted_content` 原文与该 issue 相近但不完全相同，不能直接判定为同一个缺陷。 +- 现有证据不支持从 v0.1.153 回退；回退可能重新引入已修复的 Responses 协议问题。 +- 官方来源： + - https://github.com/Wei-Shaw/sub2api/releases/tag/v0.1.153 + - https://github.com/Wei-Shaw/sub2api/issues/2067 + - https://github.com/Wei-Shaw/sub2api/pull/2068 + +## 调优建议 + +- P0：保持 v0.1.153，不回退，不提高 `max_account_switches`，不降低全池并发。 +- P0：确认实际 OpenAI 响应头/首 token 超时，把单个慢上游的允许时间收紧到能为至少一次 failover 留出预算；先做有界对照，不直接照抄固定值。 +- P1：继续使用最多 3 分钟的保守临时不可调度；当前一分钟冷却已经频繁命中，不扩大到无关键词的所有 5xx。 +- P1：针对账号 28 的 overload 和账号 27 的 502 保持即时冷却，观察完整 60 分钟窗口的客户错误率和 failover 成功率。 +- P1：对账号 33 的 `gpt-5.6-sol` continuation/`encrypted_content` 错误积累更多 request 证据；重复出现时再做账号与模型级调度隔离，不能凭单个样本做全账号摘除。 +- P1：修复账号 29、30 的余额和账号 31 的失效 API Key 后，通过真实请求验证再恢复；禁止仅改 `schedulable`。 +- P2：继续增强原生 system-log 对策略事件和最终 outcome 的索引，减少 runtime fallback 与 outcome missing。 + +## 验证 + +- `bun --check`：`runtime.ts`、`remote-python-sync.ts`、`output.ts` 通过。 +- `git diff --check` 通过。 +- `bun test scripts/src/output.test.ts`：2 pass，0 fail。 +- 默认 `runtime errors --all-groups`：紧凑表格，无 JSON envelope，无 `outputTruncated`。 +- 默认 `runtime errors --group 3`：紧凑详情、共享证据提示和 request 索引可见。 +- 默认 `runtime list`：Kubernetes 风格账号列表。 +- 显式 `--json`：返回机器可读结构。 +- degraded trace 与两条 failover-no-candidate trace 均通过真实 PK01 原生 API 回归。 + +## 后续 + +- 父任务 R2.2 保持进行中，继续按完整窗口观察调优效果。 +- 未创建外部 GitHub issue；相关未决项继续由本 MDTODO 问题域跟踪，避免未经授权写入外部系统。 diff --git a/docs/MDTODO/pr-merge-driven-automatic-delivery.md b/docs/MDTODO/pr-merge-driven-automatic-delivery.md index 161f014f..e1448d99 100644 --- a/docs/MDTODO/pr-merge-driven-automatic-delivery.md +++ b/docs/MDTODO/pr-merge-driven-automatic-delivery.md @@ -83,7 +83,7 @@ 合并修复 PR 后只观察新的 GitHub webhook→Gitea mirror→PaC→Tekton→GitOps/Argo 自动事件，验证 watcher Ready、TaskRun 创建及 AgentRun/Monitor runtime 收敛，不人工补齐既有 queued run，完成任务后将详细报告写入[任务报告](./details/pr-merge-driven-automatic-delivery/R5.3_Task_Report.md)。 -## R6 [in_progress] +## R6 [completed] 执行 [UniDesk #1981](https://github.com/pikasTech/unidesk/issues/1981)：固定仓库产品功能 schema 路径为 config/feature-config.schema.json，CI/CD 只读校验 schema 语法、值匹配和功能到唯一配置变量的一对一关系；缺失、非法、不匹配或重复仅输出 typed warning 并投影到 OTel/status/history/debug-step，必须继续 artifact、GitOps/Argo 和滚动上线，禁止写回、默认值、变量删除、功能关闭、架构切换及其他补救操作，完成任务后将详细报告写入[任务报告](./details/pr-merge-driven-automatic-delivery/R6_Task_Report.md)。 diff --git a/docs/MDTODO/sub2api-upstream-reliability.md b/docs/MDTODO/sub2api-upstream-reliability.md new file mode 100644 index 00000000..8d56aa2c --- /dev/null +++ b/docs/MDTODO/sub2api-upstream-reliability.md @@ -0,0 +1,37 @@ +# Sub2API 上游可靠性 + +本问题域持续跟踪 Sub2API 上游错误的真实归因、账号运行时可见性、受控调度缓解和用户侧错误隔离。配置事实以 owning YAML 为准，手工账号运行时配置通过受控 CRUD 显式管理。 + + +## R1 [completed] + +完善 Sub2API 上游错误归因与 runtime CRUD 可见性，并基于实际错误证据应用保守的临时不可调度策略，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R1_Task_Report.md)。 + +## R2 [in_progress] + +在新观测窗口复核临时不可调度命中、内部换号和最终客户响应，按请求分母评估调优效果并识别仍会穿透的流式错误，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2_Task_Report.md)。 + +### R2.1 [completed] + +增强 CLI 使用 Sub2API 原生 admin/ops 数据诊断请求错误、账号可用性、并发和 request trace，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.1_Task_Report.md)。 + +### R2.2 [in_progress] + +在连续新窗口复核临时不可调度、failover、客户可见错误率和流式 degraded 请求，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2_Task_Report.md)。 + +#### R2.2.1 [completed] + +溯源最近窗口客户可见错误请求，区分上游失败、切号耗尽、无候选和流式提交后错误并给出配置调优建议，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.1_Task_Report.md)。 +#### R2.2.2 [completed] + +执行客户错误调优：处理 gpt-5.4-nano 无候选，扩展 429 upstream rate limit 与精确 504 临时不可调度规则，并通过受控 runtime apply 下发和验证，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.2_Task_Report.md)。 +#### R2.2.3 [completed] + +将 runtime 精准 CRUD、批量操作、原生模型探测、客户可见错误溯源和基于真实证据的配置调优流程沉淀到 unidesk-sub2api skill，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.3_Task_Report.md)。 +#### R2.2.4 [completed] + +重新分析所有 Sub2API 分组最近的用户可见错误，并把 runtime errors 泛化为 all-groups 与精确 group 下钻；默认输出必须提供语义摘要、索引和渐进披露，禁止用手工 limit 规避输出问题，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.4_Task_Report.md)。 + +### R2.3 + +调查并收敛 Sub2API 原生 system-log 索引未保留策略事件的能力缺口，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.3_Task_Report.md)。 diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 53ac73f3..a4df1fda 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -157,8 +157,26 @@ - `runtime.templates` 保存可复用的 Sub2API runtime 规则模板。模板本身不是账号绑定，不会自动接管未在 YAML 中声明的手工账号。 - `pool.defaultTempUnschedulableTemplate` 是 YAML-managed 账号的默认临时不可调度模板引用；`profiles.entries[].tempUnschedulableTemplate` 可以为明确声明的账号选择其他模板。`codex-pool sync --confirm` 只向这些 managed 账号渲染 `temp_unschedulable_enabled` 和 `temp_unschedulable_rules`。 - YAML 未声明的账号属于 `runtime-manual`。`codex-pool runtime list|get` 可以显示其实际状态和 `matchingTemplate`，但与模板不同不构成 drift，也不得由 `sync` 自动纳管。 -- `codex-pool runtime errors` 从实际 Sub2API 日志按账号聚合错误。规则状态码只能取 `account_upstream_error` 的观测值；`openai.forward_failed` 只作为脱敏错误摘要证据，不能凭摘要推测未观测状态码。 +- `codex-pool runtime errors` 优先从 Sub2API 原生 admin/ops 能力读取服务数据： + - dashboard overview 提供请求分母、客户可见错误率和上游错误率； + - account availability 提供账号可用、限流、过载和错误状态； + - concurrency 提供组与账号容量、当前占用和排队； + - upstream errors 提供账号、状态码和根因详情； + - system-log 索引未保留策略事件时，才从 target runtime 日志补临时不可调度、failover、选号失败和 `forward_failed`，并披露 fallback； + - 规则状态码只能取 `account_upstream_error` 的观测值； + - `openai.forward_failed` 只作为脱敏错误摘要证据，不能凭摘要推测未观测状态码； + - Sub2API Ops 上游错误记录覆盖全部错误账号和状态码，默认输出全池与账号主因； + - `policyEffectiveness` 通过 `request_id` 关联临时不可调度、切号、无候选和最终网关响应，并提供可交给 `trace` 的 request id； + - 规则命中必须有 `account_temp_unschedulable` 日志证据，状态码相同不代表响应体关键词也命中； + - `requestIdCoverage` 必须披露事件关联缺口，缺少 request id 时不能用零关联证明策略链未发生； + - HTTP 200 与 `forward_failed` 可以在流式响应中并存，必须单列这类请求，不能仅凭最终 HTTP 状态判断客户收到完整终止事件； + - 窗口内最终错误占比只描述目标 API 路径的已完成日志事件，必须保留 `since`、`tail` 和分母覆盖边界； + - `--full` 才展开模型、小时、流式分布和有限详情样本； + - 账号当前并发只作为查询时快照，不能替代错误发生瞬间的历史并发证据； + - 没有账号全部路由尝试数时，只能报告错误数量，不能把错误记录数当作错误率； + - Ops 样本与 runtime 日志保持独立计数，不把时间邻近当作请求级关联。 - 手工账号的 runtime 配置必须通过 `codex-pool runtime apply|delete` 受控修改。写操作必须显式指定 `--account`，不带 `--confirm` 时只生成 dry-run，带 `--confirm` 才通过 Sub2API admin API 写入。 +- `codex-pool trace` 优先使用原生 `/admin/ops/requests` 和 `/admin/ops/system-logs`，target runtime 日志只补索引缺失的邻近事件；必须显式披露 target、runtime mode、主数据源、上下文数据源和读取状态。PK01 host-Docker 回退时不得固定读取 k3s Deployment，并且必须合并 Docker stdout/stderr 日志流。HTTP 200 与 `forward_failed` 并存且没有 failover 时应标记为 `degraded`。 - 内置临时不可调度配置与外部 `sentinel.*` 是独立控制面。runtime 模板负责请求路径即时冷却与 failover；`sentinel.*` 负责账号 marker health、隔离、恢复和 probe 周期，任一控制面都不得静默重写另一控制面的状态或规则。 - The external sentinel write surface is intentionally limited to the Sub2API admin `schedulable` action. Sentinel freeze/restore may set `schedulable=false|true`, but must not write, clear, or indirectly clear Sub2API request-path runtime state such as `temp_unschedulable_until`, `temp_unschedulable_reason`, rate-limit, overload, or model-rate-limit state. In particular, sentinel restore must not call Sub2API `recover-state`, because that endpoint is a broader runtime-state recovery operation rather than a pure schedulability restore. - Codex accounts selected by YAML do not declare `schedulable` as durable configuration. `codex-pool sync --confirm` must not restore existing account schedulability merely because YAML selects the account or sentinel state lacks an active quarantine. Existing `schedulable=false` is runtime state: the sentinel first reads Sub2API's actual account state, schedules a recovery probe for unschedulable managed accounts, and restores `schedulable=true` only after the marker probe matches. diff --git a/scripts/src/output.ts b/scripts/src/output.ts index 4c217e03..2cc8202c 100644 --- a/scripts/src/output.ts +++ b/scripts/src/output.ts @@ -307,7 +307,7 @@ function renderTextOutput(command: string, text: string): string { trigger, ...(dump === null ? {} : { dump }), next: [ - "Use rg --max-count, head, tail, --limit, --tail-bytes, or an id-specific query.", + "Use the command's semantic summary, page cursor, stable id, or id-specific detail query before requesting complete output.", "Use --full or --raw only when the complete one-off payload is intentionally required.", ], }, @@ -341,7 +341,7 @@ function oversizedOutputNext(envelope: JsonEnvelope): string[] { ]; } return [ - "Use --limit, --tail-bytes, an id-specific view, or another semantic narrow query.", + "Use the command's semantic summary, page cursor, stable id, or id-specific detail query; improve the command when none is available.", "Use --full or --raw only when the complete one-off payload is intentionally required.", ]; } diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts index 4facf494..88c75360 100644 --- a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts @@ -2867,6 +2867,11 @@ def classify_trace_event(item, account_names_by_id): "error": item.get("error"), "excludedAccountCount": item.get("excluded_account_count"), }) + elif "openai.forward_failed" in message: + event.update({ + "type": "forward-failed", + "error": item.get("error"), + }) elif "account_upstream_error" in message: event.update({ "type": "upstream-error", @@ -2939,10 +2944,15 @@ def trace_reason(events, final_event): failovers = [item for item in events if item.get("type") == "failover"] select_failures = [item for item in events if item.get("type") == "select-failed"] upstream_errors = [item for item in events if item.get("type") == "upstream-error"] + forward_failures = [item for item in events if item.get("type") == "forward-failed"] if failover_budget_exhausted(failovers, final_event): return "failover-budget-exhausted" if failovers and select_failures: return "failover-attempted-no-candidate" + if forward_failures and isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400: + return "forward-failed-http-success" + if forward_failures: + return "forward-failed" if failovers: return "failover-attempted" if select_failures: @@ -3006,6 +3016,60 @@ def run_trace(): if not isinstance(request_id, str) or not request_id: raise RuntimeError("trace payload missing requestId") admin_email, token, admin_compliance = login() + request_query = quote(request_id, safe="") + native_request_status = {"status": "unavailable"} + native_request = None + try: + request_data = ensure_success( + curl_api("GET", f"/api/v1/admin/ops/requests?request_id={request_query}&kind=all&page=1&page_size=100", bearer=token), + "get ops request details", + ) + request_items = extract_items(request_data) + native_request = next((item for item in request_items if isinstance(item, dict) and item.get("request_id") == request_id), None) + native_request_status = { + "status": "available", + "matched": native_request is not None, + "returned": len(request_items), + "total": request_data.get("total", len(request_items)) if isinstance(request_data, dict) else len(request_items), + } + except Exception as exc: + native_request_status = {"status": "unavailable", "reason": text(str(exc), 500)} + + native_system_log_status = {"status": "unavailable"} + native_matched = [] + try: + system_log_data = ensure_success( + curl_api("GET", f"/api/v1/admin/ops/system-logs?request_id={request_query}&page=1&page_size=200", bearer=token), + "get ops request system logs", + ) + system_log_items = extract_items(system_log_data) + for row in system_log_items: + if not isinstance(row, dict): + continue + parsed = dict(row.get("extra") or {}) if isinstance(row.get("extra"), dict) else {} + for source_key in ("request_id", "client_request_id", "account_id", "platform", "model"): + if parsed.get(source_key) is None and row.get(source_key) is not None: + parsed[source_key] = row.get(source_key) + parsed["_at"] = row.get("created_at") + parsed["_message"] = row.get("message") + parsed["_line"] = json.dumps({ + "created_at": row.get("created_at"), + "message": row.get("message"), + "request_id": row.get("request_id"), + "account_id": row.get("account_id"), + "platform": row.get("platform"), + "model": row.get("model"), + "extra": parsed, + }, ensure_ascii=False, default=str) + native_matched.append(parsed) + native_system_log_status = { + "status": "available", + "returned": len(system_log_items), + "total": system_log_data.get("total", len(system_log_items)) if isinstance(system_log_data, dict) else len(system_log_items), + } + except Exception as exc: + native_system_log_status = {"status": "unavailable", "reason": text(str(exc), 500)} + account_snapshot, account_snapshot_error = account_snapshot_from_runtime(token) account_names_by_id = {} for row in account_snapshot: @@ -3014,17 +3078,21 @@ def run_trace(): account_id = int(account_id) if isinstance(account_id, int) and isinstance(row.get("accountName"), str): account_names_by_id[account_id] = row.get("accountName") - proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"]) - stdout = proc.stdout.decode("utf-8", errors="replace") + proc = runtime_logs(since, tail) + log_bytes = proc.stdout + b"\\n" + proc.stderr + stdout = log_bytes.decode("utf-8", errors="replace") parsed_lines = [] - matched = [] + runtime_matched = [] for line in stdout.splitlines(): parsed = parse_log_line(line) if parsed is None: continue parsed_lines.append(parsed) if request_id in line: - matched.append(parsed) + runtime_matched.append(parsed) + matched = native_matched if native_matched else runtime_matched + matched.sort(key=lambda item: (log_time_epoch(item) is None, log_time_epoch(item) or 0)) + trace_event_source = "sub2api-native-admin-ops-system-logs" if native_matched else "runtime-logs-fallback" first_epoch = None last_epoch = None for item in matched: @@ -3045,22 +3113,55 @@ def run_trace(): window_lines = matched events = [classify_trace_event(item, account_names_by_id) for item in matched] request_start = next((item for item in events if item.get("type") == "request-start"), None) + if isinstance(native_request, dict): + if not isinstance(request_start, dict): + request_start = { + "type": "request-start", + "at": native_request.get("created_at"), + "requestId": native_request.get("request_id"), + "path": None, + "model": native_request.get("model"), + "accountId": native_request.get("account_id"), + "accountName": account_names_by_id.get(native_request.get("account_id")), + "stream": native_request.get("stream"), + "source": "sub2api-native-admin-ops-requests", + } + else: + if request_start.get("stream") is None: + request_start["stream"] = native_request.get("stream") + if not request_start.get("model"): + request_start["model"] = native_request.get("model") final_event = next((item for item in reversed(events) if item.get("type") == "final"), None) + if not isinstance(final_event, dict) and isinstance(native_request, dict) and isinstance(native_request.get("status_code"), int): + final_event = { + "type": "final", + "at": native_request.get("created_at"), + "requestId": native_request.get("request_id"), + "path": None, + "model": native_request.get("model"), + "accountId": native_request.get("account_id"), + "accountName": account_names_by_id.get(native_request.get("account_id")), + "statusCode": native_request.get("status_code"), + "latencyMs": native_request.get("duration_ms"), + "source": "sub2api-native-admin-ops-requests", + } failovers = [item for item in events if item.get("type") == "failover"] select_failures = [item for item in events if item.get("type") == "select-failed"] upstream_errors = [item for item in events if item.get("type") == "upstream-error"] + forward_failures = [item for item in events if item.get("type") == "forward-failed"] temp_unsched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if "account_temp_unschedulable" in str(item.get("_message") or "")] admin_sched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if ("schedulable" in str(item.get("_message") or "") or "/schedulable" in str(item.get("path") or ""))] window_events = [classify_trace_event(item, account_names_by_id) for item in window_lines] final_errors = [item for item in window_events if item.get("type") == "final" and isinstance(item.get("statusCode"), int) and item.get("statusCode") >= 400] window_failovers = [item for item in window_events if item.get("type") == "failover"] window_select_failures = [item for item in window_events if item.get("type") == "select-failed"] + window_forward_failures = [item for item in window_events if item.get("type") == "forward-failed"] untried_schedulable_accounts = trace_untried_schedulable_accounts(failovers, final_event or {}, account_snapshot) reason = trace_reason(events, final_event) if not matched: outcome = "not-found" elif isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400: - outcome = "succeeded" + outcome = "degraded" if forward_failures else ("succeeded-with-failover" if failovers else "succeeded") elif isinstance(final_event, dict): outcome = "failed" else: @@ -3068,11 +3169,14 @@ def run_trace(): return { "ok": proc.returncode == 0 and len(matched) > 0, "mode": "trace", + "targetId": TARGET_ID, + "runtimeMode": RUNTIME_MODE, "namespace": NAMESPACE, "serviceDns": SERVICE_DNS, "appPod": APP_POD, "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, "requestId": request_id, + "eventSource": trace_event_source, "summary": { "outcome": outcome, "reason": reason, @@ -3090,10 +3194,16 @@ def run_trace(): }, "request": request_start or {}, "final": final_event or {}, + "nativeOps": { + "requestDetails": native_request_status, + "request": native_request, + "systemLogs": native_system_log_status, + }, "events": events, "failovers": failovers, "selectFailures": select_failures, "upstreamErrors": upstream_errors, + "forwardFailures": forward_failures, "tempUnschedulable": temp_unsched, "adminSchedulable": admin_sched[-20:], "windowStats": { @@ -3102,6 +3212,8 @@ def run_trace(): "finalErrorCount": len(final_errors), "failoverCount": len(window_failovers), "selectFailedCount": len(window_select_failures), + "forwardFailedCount": len(window_forward_failures), + "matchedForwardFailedCount": len(forward_failures), "tempUnschedulableCount": len(temp_unsched), "adminSchedulableCount": len(admin_sched), }, @@ -3114,8 +3226,10 @@ def run_trace(): "rawLines": [{"line": item.get("_line")} for item in matched[-30:]] if show_lines else [], "showLines": show_lines, "logs": { + "source": trace_event_source, + "contextSource": f"host-docker:{HOST_DOCKER_APP_CONTAINER}" if RUNTIME_MODE == "host-docker" else f"k3s:{NAMESPACE}/deployment/sub2api", "exitCode": proc.returncode, - "stderrTail": text(proc.stderr, 1000), + "stderrTail": text(proc.stderr, 1000) if proc.returncode != 0 else "", "stdoutLineCount": len(stdout.splitlines()), }, "valuesPrinted": False, diff --git a/scripts/src/platform-infra-sub2api-codex/render.ts b/scripts/src/platform-infra-sub2api-codex/render.ts index a8898fc0..a7052e38 100644 --- a/scripts/src/platform-infra-sub2api-codex/render.ts +++ b/scripts/src/platform-infra-sub2api-codex/render.ts @@ -398,8 +398,10 @@ export function renderTraceReport( const failovers = recordArray(parsed.failovers); const selectFailures = recordArray(parsed.selectFailures); const upstreamErrors = recordArray(parsed.upstreamErrors); + const forwardFailures = recordArray(parsed.forwardFailures); const tempUnschedulable = recordArray(parsed.tempUnschedulable); const windowStats = isRecord(parsed.windowStats) ? parsed.windowStats : {}; + const logs = isRecord(parsed.logs) ? parsed.logs : {}; const accountSnapshot = recordArray(parsed.accountSnapshot); const lines: string[] = []; lines.push([ @@ -409,6 +411,17 @@ export function renderTraceReport( `outcome=${stringValue(summary.outcome) ?? "-"}`, `reason=${stringValue(summary.reason) ?? "-"}`, ].join(" ")); + lines.push([ + "SOURCE", + `target=${textValue(parsed.targetId)}`, + `runtime=${textValue(parsed.runtimeMode)}`, + `logs=${textValue(logs.source)}`, + `context=${textValue(logs.contextSource)}`, + `exit=${textValue(logs.exitCode)}`, + ].join(" ")); + if (parsed.ok !== true && stringValue(logs.stderrTail) !== null) { + lines.push(`LOG ERROR ${shorten(stringValue(logs.stderrTail) ?? "", 240)}`); + } lines.push([ "REQUEST", `path=${request.path ?? final.path ?? "-"}`, @@ -452,6 +465,19 @@ export function renderTraceReport( ]), ])); } + if (forwardFailures.length > 0) { + lines.push(""); + lines.push("FORWARD-FAILED"); + lines.push(renderTable([ + ["AT", "ACCOUNT", "STATUS", "ERROR"], + ...forwardFailures.map((item) => [ + shortIso(item.at), + formatAccountRef(item), + textValue(item.statusCode), + shorten(stringValue(item.error) ?? "-", 72), + ]), + ])); + } if (upstreamErrors.length > 0 || tempUnschedulable.length > 0) { lines.push(""); lines.push("ACCOUNT SIGNALS"); @@ -482,13 +508,14 @@ export function renderTraceReport( lines.push(""); lines.push("WINDOW STATS"); lines.push(renderTable([ - ["MATCH", "EVENTS", "FINAL_4XX_5XX", "FAILOVER", "SELECT_FAIL", "TEMP_UNSCHED", "ADMIN_SCHED"], + ["MATCH", "EVENTS", "FINAL_4XX_5XX", "FAILOVER", "SELECT_FAIL", "FORWARD_FAIL", "TEMP_UNSCHED", "ADMIN_SCHED"], [ textValue(windowStats.matchedLines), textValue(windowStats.eventCount), textValue(windowStats.finalErrorCount), textValue(windowStats.failoverCount), textValue(windowStats.selectFailedCount), + textValue(windowStats.forwardFailedCount), textValue(windowStats.tempUnschedulableCount), textValue(windowStats.adminSchedulableCount), ], diff --git a/scripts/src/platform-infra-sub2api-codex/runtime.ts b/scripts/src/platform-infra-sub2api-codex/runtime.ts index e5f1b561..103ca8b0 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime.ts @@ -1,10 +1,12 @@ import type { UniDeskConfig } from "../config"; +import type { RenderedCliResult } from "../output"; import { desiredAccountNames } from "./accounts"; import { readCodexPoolConfig } from "./config"; import { protectedManualAccountsForTarget } from "./public-exposure"; import { renderSub2ApiTempUnschedulableCredentials } from "./redaction"; import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote"; +import { renderedCliResult, renderTable } from "./render"; import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target"; type RuntimeAction = "list" | "get" | "errors" | "apply" | "delete"; @@ -12,17 +14,22 @@ type RuntimeAction = "list" | "get" | "errors" | "apply" | "delete"; interface RuntimeOptions { action: RuntimeAction; account: string | null; + group: string | null; + allGroups: boolean; + platform: string | null; + pageToken: string | null; template: string | null; kind: "temp-unschedulable"; confirm: boolean; full: boolean; raw: boolean; + json: boolean; since: string; tail: number; targetId: string; } -export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise> { +export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { const options = parseRuntimeOptions(args); const pool = readCodexPoolConfig(); const target = codexPoolRuntimeTarget(options.targetId); @@ -33,6 +40,10 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P const payload = { action: options.action, account: options.account, + group: options.group, + allGroups: options.allGroups, + platform: options.platform, + pageToken: options.pageToken, kind: options.kind, confirm: options.confirm, full: options.full, @@ -61,12 +72,12 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P return { ok, action: "platform-infra-sub2api-codex-pool-runtime", - remote: compactCapture(result, { full: true }), + remote: compactCapture(result, { full: parsed === null || result.exitCode !== 0 }), parsed, valuesPrinted: false, }; } - return { + const response = { ok, action: "platform-infra-sub2api-codex-pool-runtime", target: { @@ -79,14 +90,18 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P request: { operation: options.action, account: options.account, + group: options.group, + allGroups: options.allGroups, + platform: options.platform, + pageToken: options.pageToken, template: options.template, kind: options.kind, confirm: options.confirm, since: options.action === "errors" ? options.since : undefined, tail: options.action === "errors" ? options.tail : undefined, }, - runtime: parsed, - remote: compactCapture(result, { full: options.full || result.exitCode !== 0 || parsed === null }), + runtime: compactRuntimeErrors(parsed, options), + remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }), disclosure: options.action === "list" ? { get: "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account ", @@ -95,19 +110,327 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P : undefined, valuesPrinted: false, }; + return options.full || options.json ? response : renderRuntimeResult(response, options); +} + +function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown { + if (options.action !== "errors" || options.full || value === null || typeof value !== "object" || Array.isArray(value)) return value; + const runtime = value as Record; + const allGroups = runtimeRecord(runtime.allGroups); + if (allGroups !== null) { + const nextPageToken = typeof allGroups.nextPageToken === "string" ? allGroups.nextPageToken : null; + const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; + const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; + return { + ok: runtime.ok, + operation: runtime.operation, + mutation: runtime.mutation, + endpoint: runtime.endpoint, + allGroups, + disclosure: { + group: `${baseCommand} --group ${platformFlag}`, + nextPage: nextPageToken === null ? null : `${baseCommand} --all-groups${platformFlag} --page-token ${nextPageToken}`, + note: "The overview uses stable group-id pagination. Use group drilldown for account root causes and request indexes.", + }, + valuesPrinted: false, + }; + } + const errors = runtimeRecord(runtime.errors); + if (errors === null) return value; + const effectiveness = runtimeRecord(errors.policyEffectiveness) ?? {}; + const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; + const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; + const availabilityGroup = runtimeRecord(availability.group) ?? {}; + const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; + const concurrencyGroup = runtimeRecord(concurrency.group) ?? {}; + const sink = runtimeRecord(nativeOps.systemLogSink) ?? {}; + const sourceStatus = runtimeRecord(effectiveness.sourceStatus) ?? {}; + const drilldown = runtimeRecord(effectiveness.requestDrilldown) ?? {}; + const requestIndexes = Object.fromEntries(Object.entries(drilldown).flatMap(([key, item]) => { + if (!Array.isArray(item)) return []; + const ids = item.filter((entry): entry is string => typeof entry === "string"); + if (ids.length === 0) return []; + return [[key, { + count: ids.length, + sampleRequestId: ids[0], + moreAvailable: ids.length > 1, + }]]; + })); + const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; + const groupFlag = options.group === null ? "" : ` --group ${options.group}`; + const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; + return { + ok: runtime.ok, + operation: runtime.operation, + mutation: runtime.mutation, + group: runtime.group, + endpoint: runtime.endpoint, + errors: { + window: errors.window, + nativeOps: { + source: nativeOps.source, + overview: nativeOps.overview, + accountAvailability: { + status: availability.status, + totalAccounts: availabilityGroup.total_accounts, + availableCount: availabilityGroup.available_count, + rateLimitCount: availabilityGroup.rate_limit_count, + errorCount: availabilityGroup.error_count, + unavailableAccounts: Array.isArray(availability.unavailableAccounts) + ? availability.unavailableAccounts.map((item) => { + const account = runtimeRecord(item) ?? {}; + return { + accountId: account.accountId, + accountName: account.accountName, + status: account.status, + reason: account.error, + }; + }) + : [], + }, + concurrency: { + status: concurrency.status, + currentInUse: concurrencyGroup.current_in_use, + maxCapacity: concurrencyGroup.max_capacity, + loadPercentage: concurrencyGroup.load_percentage, + waitingInQueue: concurrencyGroup.waiting_in_queue, + }, + systemLogSink: { + status: sink.status, + queueDepth: sink.queueDepth, + droppedCount: sink.droppedCount, + writeFailedCount: sink.writeFailedCount, + }, + }, + rootCauseAnalysis: errors.rootCauseAnalysis, + policyEffectiveness: { + source: effectiveness.source, + scope: effectiveness.scope, + attribution: effectiveness.attribution, + assessment: effectiveness.assessment, + ruleMatches: effectiveness.ruleMatches, + failover: effectiveness.failover, + forwardFailure: effectiveness.forwardFailure, + selectFailedEvents: effectiveness.selectFailedEvents, + requestIdCoverage: effectiveness.requestIdCoverage, + gatewayOutcomeEvidence: effectiveness.gatewayOutcomeEvidence, + requestIndexes, + sourceStatus: { + status: sourceStatus.status, + source: sourceStatus.source, + fallbackUsed: sourceStatus.fallbackUsed, + fallbackReason: sourceStatus.fallbackReason, + fallbackEventCount: sourceStatus.fallbackEventCount, + }, + }, + }, + disclosure: { + complete: `${baseCommand}${groupFlag}${platformFlag}${options.account === null ? "" : " --account "} --full`, + account: `${baseCommand}${groupFlag}${platformFlag} --account `, + trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", + note: "Use requestIndexes for trace drilldown and --full for the complete finite index.", + }, + valuesPrinted: false, + }; +} + +function runtimeRecord(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : null; +} + +function runtimeRecords(value: unknown): Record[] { + return Array.isArray(value) ? value.flatMap((item) => { + const record = runtimeRecord(item); + return record === null ? [] : [record]; + }) : []; +} + +function runtimeText(value: unknown): string { + if (value === null || value === undefined || value === "") return "-"; + if (typeof value === "boolean") return value ? "true" : "false"; + if (Array.isArray(value)) return value.map(runtimeText).join(","); + return String(value).replace(/\s+/gu, " ").trim(); +} + +function runtimeShort(value: unknown, max = 42): string { + const text = runtimeText(value); + return text.length <= max ? text : `${text.slice(0, max - 1)}…`; +} + +function runtimeRate(value: unknown): string { + return typeof value === "number" && Number.isFinite(value) ? `${(value * 100).toFixed(2)}%` : "-"; +} + +function renderRuntimeResult(response: Record, options: RuntimeOptions): RenderedCliResult { + const runtime = runtimeRecord(response.runtime) ?? {}; + const ok = response.ok === true; + const lines: string[] = []; + if (options.action === "errors") renderRuntimeErrors(lines, runtime, options); + else if (options.action === "list") renderRuntimeAccountList(lines, runtime); + else if (options.action === "get") renderRuntimeAccountDetail(lines, runtime); + else renderRuntimeMutation(lines, runtime, options); + lines.push("", `JSON: bun scripts/cli.ts platform-infra sub2api codex-pool runtime ${options.action} --json`); + return renderedCliResult(ok, `platform-infra sub2api codex-pool runtime ${options.action}`, lines.join("\n")); +} + +function renderRuntimeErrors(lines: string[], runtime: Record, options: RuntimeOptions): void { + const allGroups = runtimeRecord(runtime.allGroups); + if (allGroups !== null) { + const totals = runtimeRecord(allGroups.pageTotals) ?? {}; + const groups = runtimeRecords(allGroups.groups); + lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=all-groups since=${options.since}`); + lines.push(renderTable([ + ["GROUPS", "RETURNED", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UPSTREAM_ERR", "UP_RATE", "ATTENTION"], + [ + runtimeText(allGroups.totalGroups), runtimeText(allGroups.returnedGroups), runtimeText(totals.requestCount), + runtimeText(totals.errorCount), runtimeRate(totals.errorRate), runtimeText(totals.upstreamErrorCount), + runtimeRate(totals.upstreamErrorRate), runtimeText(totals.groupsNeedingAttention), + ], + ])); + lines.push("", "GROUPS"); + lines.push(renderTable([ + ["NAME", "ID", "PLATFORM", "STATUS", "REQ", "CLIENT_ERR", "ERR_RATE", "UP_ERR", "UP_RATE", "ACCOUNTS", "AVAILABLE", "USE/CAP", "QUEUE", "ATTN"], + ...groups.map((group) => [ + runtimeShort(group.groupName, 28), runtimeText(group.groupId), runtimeText(group.platform), runtimeText(group.status), + runtimeText(group.requestCount), runtimeText(group.errorCount), runtimeRate(group.errorRate), runtimeText(group.upstreamErrorCount), + runtimeRate(group.upstreamErrorRate), runtimeText(group.totalAccounts), runtimeText(group.availableCount), + `${runtimeText(group.currentInUse)}/${runtimeText(group.maxCapacity)}`, runtimeText(group.waitingInQueue), group.needsAttention === true ? "yes" : "no", + ]), + ])); + const disclosure = runtimeRecord(runtime.disclosure) ?? {}; + lines.push("", `Detail: ${runtimeText(disclosure.group)}`); + if (typeof disclosure.nextPage === "string") lines.push(`Next: ${disclosure.nextPage}`); + return; + } + const group = runtimeRecord(runtime.group) ?? {}; + const errors = runtimeRecord(runtime.errors) ?? {}; + const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; + const overview = runtimeRecord(nativeOps.overview) ?? {}; + const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; + const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; + lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`); + lines.push(renderTable([ + ["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"], + [ + runtimeShort(group.name, 30), runtimeText(group.id), runtimeText(group.platform), runtimeText(overview.requestCount), + runtimeText(overview.errorCount), runtimeRate(overview.errorRate), runtimeRate(overview.upstreamErrorRate), + runtimeText(overview.businessLimitedCount), runtimeText(overview.healthScore), + ], + ])); + lines.push(renderTable([ + ["ACCOUNTS", "AVAILABLE", "RATE_LIMIT", "ERROR", "IN_USE", "CAPACITY", "QUEUE"], + [ + runtimeText(availability.totalAccounts), runtimeText(availability.availableCount), runtimeText(availability.rateLimitCount), + runtimeText(availability.errorCount), runtimeText(concurrency.currentInUse), runtimeText(concurrency.maxCapacity), runtimeText(concurrency.waitingInQueue), + ], + ])); + const unavailable = runtimeRecords(availability.unavailableAccounts); + if (unavailable.length > 0) { + lines.push("", "UNAVAILABLE ACCOUNTS", renderTable([ + ["ACCOUNT", "ID", "STATUS", "REASON"], + ...unavailable.map((account) => [runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.status), runtimeShort(account.reason, 70)]), + ])); + } + const root = runtimeRecord(errors.rootCauseAnalysis) ?? {}; + const causes = runtimeRecords(root.causeBuckets); + if (causes.length > 0) { + lines.push("", `ROOT CAUSES scope=${runtimeText(root.scope)}`, renderTable([ + ["CAUSE", "COUNT"], + ...causes.map((cause) => [runtimeText(cause.cause), runtimeText(cause.count)]), + ])); + } + const policy = runtimeRecord(errors.policyEffectiveness) ?? {}; + const rules = runtimeRecord(policy.ruleMatches) ?? {}; + const failover = runtimeRecord(policy.failover) ?? {}; + const forward = runtimeRecord(policy.forwardFailure) ?? {}; + lines.push("", `POLICY EFFECTIVENESS scope=${runtimeText(policy.scope)}`); + lines.push(renderTable([ + ["TEMP_UNSCHED", "FAILOVER", "SUCCESS", "FAILED", "MISSING", "FORWARD_200", "FORWARD_ERR", "SELECT_FAIL"], + [ + runtimeText(rules.tempUnschedulableEvents), runtimeText(failover.requests), runtimeText(failover.succeeded), + runtimeText(failover.failed), runtimeText(failover.outcomeMissing), runtimeText(forward.withHttpSuccess), + runtimeText(forward.withHttpError), runtimeText(policy.selectFailedEvents), + ], + ])); + const indexes = runtimeRecord(policy.requestIndexes) ?? {}; + const indexRows = Object.entries(indexes).flatMap(([kind, value]) => { + const item = runtimeRecord(value); + return item === null ? [] : [[kind, runtimeText(item.count), runtimeText(item.sampleRequestId), item.moreAvailable === true ? "yes" : "no"]]; + }); + if (indexRows.length > 0) lines.push("", "REQUEST INDEX", renderTable([["TYPE", "COUNT", "SAMPLE_REQUEST_ID", "MORE"], ...indexRows])); + if (typeof root.attribution === "string") lines.push("", `NOTE: ${root.attribution}`); + if (typeof policy.attribution === "string") lines.push(`NOTE: ${policy.attribution}`); + const disclosure = runtimeRecord(runtime.disclosure) ?? {}; + lines.push("", `Account: ${runtimeText(disclosure.account)}`, `Trace: ${runtimeText(disclosure.trace)}`, `Full: ${runtimeText(disclosure.complete)}`); +} + +function renderRuntimeAccountList(lines: string[], runtime: Record): void { + const group = runtimeRecord(runtime.group) ?? {}; + const accounts = runtimeRecords(runtime.accounts); + lines.push(`SUB2API RUNTIME ACCOUNTS ok=${runtime.ok === true ? "true" : "false"} group=${runtimeText(group.name)} count=${accounts.length}`); + lines.push(renderTable([ + ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "CONCURRENCY", "LOAD", "PRIORITY", "TEMP_RULES", "TEMPLATE"], + ...accounts.map((account) => { + const temp = runtimeRecord(account.tempUnschedulable) ?? {}; + return [ + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), + runtimeText(account.schedulable), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, + runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(temp.ruleCount), runtimeText(account.matchingTemplate), + ]; + }), + ])); + lines.push("", "Detail: bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account "); +} + +function renderRuntimeAccountDetail(lines: string[], runtime: Record): void { + const account = runtimeRecord(runtime.account) ?? {}; + const temp = runtimeRecord(account.tempUnschedulable) ?? {}; + lines.push(`SUB2API RUNTIME ACCOUNT ok=${runtime.ok === true ? "true" : "false"}`); + lines.push(renderTable([ + ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "PROXY", "CONCURRENCY", "LOAD", "PRIORITY", "TEMPLATE"], + [ + runtimeShort(account.accountName, 42), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), + runtimeText(account.schedulable), runtimeText(account.proxyId), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, + runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(account.matchingTemplate), + ], + ])); + lines.push(renderTable([ + ["TEMP_ENABLED", "RULES", "STATUS_CODES", "POOL_MODE"], + [runtimeText(temp.enabled), runtimeText(temp.ruleCount), runtimeText(temp.statusCodes), runtimeText(account.poolModeEnabled)], + ])); +} + +function renderRuntimeMutation(lines: string[], runtime: Record, options: RuntimeOptions): void { + const plan = runtimeRecord(runtime.plan) ?? {}; + const account = runtimeRecord(plan.account) ?? runtimeRecord(runtime.account) ?? {}; + const before = runtimeRecord(plan.before) ?? {}; + const after = runtimeRecord(plan.after) ?? {}; + lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); + lines.push(renderTable([ + ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES"], + [ + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), + runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), + ], + ])); } export function parseRuntimeOptions(args: string[]): RuntimeOptions { const [actionRaw = "list", ...rest] = args; if (actionRaw !== "list" && actionRaw !== "get" && actionRaw !== "errors" && actionRaw !== "apply" && actionRaw !== "delete") { - throw new Error("runtime usage: list|get|errors|apply|delete [--account ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--full|--raw]"); + throw new Error("runtime usage: list|get|errors|apply|delete [--account ] [--group |--all-groups] [--platform ] [--page-token ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--json|--full|--raw]"); } let account: string | null = null; + let group: string | null = null; + let allGroups = false; + let platform: string | null = null; + let pageToken: string | null = null; let template: string | null = null; let kind = "temp-unschedulable" as const; let confirm = false; let full = false; let raw = false; + let json = false; let since = "24h"; let tail = 50_000; let targetId = defaultCodexPoolRuntimeTargetId(); @@ -122,8 +445,17 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { if (arg === "--confirm") confirm = true; else if (arg === "--full") full = true; else if (arg === "--raw") raw = true; + else if (arg === "--json") json = true; + else if (arg === "-o" && readValue("-o") === "json") json = true; else if (arg === "--account") account = readValue("--account"); else if (arg.startsWith("--account=")) account = arg.slice("--account=".length).trim(); + else if (arg === "--group") group = readValue("--group"); + else if (arg.startsWith("--group=")) group = arg.slice("--group=".length).trim(); + else if (arg === "--all-groups") allGroups = true; + else if (arg === "--platform") platform = readValue("--platform"); + else if (arg.startsWith("--platform=")) platform = arg.slice("--platform=".length).trim(); + else if (arg === "--page-token") pageToken = readValue("--page-token"); + else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim(); else if (arg === "--template") template = readValue("--template"); else if (arg.startsWith("--template=")) template = arg.slice("--template=".length).trim(); else if (arg === "--kind") kind = parseRuntimeKind(readValue("--kind")); @@ -140,13 +472,20 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { throw new Error(`runtime ${actionRaw} requires --account `); } if (account !== null && (account.length > 256 || /[\r\n]/u.test(account))) throw new Error("--account has an unsupported format"); + if (group !== null && (group.length > 256 || /[\r\n]/u.test(group))) throw new Error("--group has an unsupported format"); + if (platform !== null && (!/^[A-Za-z0-9._-]+$/u.test(platform) || platform.length > 64)) throw new Error("--platform has an unsupported format"); + if (pageToken !== null && (!/^\d+$/u.test(pageToken) || pageToken.length > 20)) throw new Error("--page-token must be a numeric group cursor"); + if (group !== null && allGroups) throw new Error("use only one of --group or --all-groups"); + if ((group !== null || allGroups || platform !== null || pageToken !== null) && actionRaw !== "errors") throw new Error(`runtime ${actionRaw} does not accept group scope options`); + if (pageToken !== null && !allGroups) throw new Error("--page-token requires --all-groups"); + if (account !== null && allGroups) throw new Error("--account cannot be combined with --all-groups; use --group first"); if (actionRaw === "apply" && !template) throw new Error("runtime apply requires --template "); if (actionRaw !== "apply" && template !== null) throw new Error(`runtime ${actionRaw} does not accept --template`); if (actionRaw !== "errors" && (since !== "24h" || tail !== 50_000)) throw new Error(`runtime ${actionRaw} does not accept --since or --tail`); if (!/^\d+[mhd]$/u.test(since)) throw new Error("--since must use m, h, or d"); if ((actionRaw === "list" || actionRaw === "get" || actionRaw === "errors") && confirm) throw new Error(`runtime ${actionRaw} does not accept --confirm`); - if (full && raw) throw new Error("use only one of --full or --raw"); - return { action: actionRaw, account, template, kind, confirm, full, raw, since, tail, targetId }; + if ([full, raw, json].filter(Boolean).length > 1) throw new Error("use only one of --json, --full, or --raw"); + return { action: actionRaw, account, group, allGroups, platform, pageToken, template, kind, confirm, full, raw, json, since, tail, targetId }; } function parseRuntimeKind(value: string): "temp-unschedulable" { @@ -172,6 +511,7 @@ import json import re import subprocess import sys +from datetime import datetime, timedelta, timezone from urllib.parse import quote RUNTIME_MODE = ${pyJson(target.runtimeMode)} @@ -324,11 +664,14 @@ def login(): raise RuntimeError("admin login response has no token") return token -def list_groups(token): - return items_of(data_of(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups")) +def list_groups(token, platform=None): + suffix = "?platform=" + quote(str(platform), safe="") if platform else "" + return items_of(data_of(curl_api("GET", "/api/v1/admin/groups/all" + suffix, bearer=token), "list groups")) -def list_group_accounts(token, group_id): - path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500&platform=openai" +def list_group_accounts(token, group_id, platform=None): + path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500" + if platform: + path += "&platform=" + quote(str(platform), safe="") return items_of(data_of(curl_api("GET", path, bearer=token), "list group accounts")) def account_detail(token, account_id): @@ -372,6 +715,10 @@ def account_summary(detail, include_rules=False): "status": detail.get("status"), "schedulable": detail.get("schedulable"), "proxyId": detail.get("proxy_id"), + "concurrencyLimit": detail.get("concurrency"), + "currentConcurrency": detail.get("current_concurrency"), + "loadFactor": detail.get("load_factor"), + "priority": detail.get("priority"), "tempUnschedulable": {key: policy[key] for key in ("enabled", "ruleCount", "statusCodes")}, "matchingTemplate": matching_template(credentials), "poolModeEnabled": credentials.get("pool_mode") is True, @@ -412,15 +759,676 @@ def compact_error(value): compact = re.sub(r"(?i)\\b(api[_ -]?key|token)[=:]\\s*[^\\s,;]+", r"\\1=[REDACTED]", compact) return compact[:240] -def observed_runtime_errors(accounts): +def since_start_time(value): + match = re.fullmatch(r"(\\d+)([mhd])", str(value or "")) + if not match: + raise RuntimeError("unsupported runtime error window: " + str(value)) + amount = int(match.group(1)) + unit = match.group(2) + duration = timedelta(minutes=amount) if unit == "m" else timedelta(hours=amount) if unit == "h" else timedelta(days=amount) + return (datetime.now(timezone.utc) - duration).isoformat().replace("+00:00", "Z") + +def ops_error_cause(value): + lower = value.lower() + markers = { + "concurrency": ("concurrency limit", "concurrent request", "too many concurrent", "maximum concurrency", "max concurrency"), + "overload": ("overloaded", "server busy", "temporarily unavailable", "capacity exhausted"), + "model-route-unavailable": ("no available channel for model", "unsupported model", "model not found", "model_not_found"), + "context-window": ("context window", "context length", "maximum context"), + "continuation-payload": ("encrypted_content", "missing_required_parameter", "missing required parameter"), + "invalid-request": ("invalid_request", "invalid request"), + "rate-limit": ("rate limit", "too many requests"), + "access-denied": ("forbidden", "access denied", "permission denied"), + "not-found": ("not found", "does not exist"), + "incomplete-stream": ("ended before a terminal event", "missing terminal event", "before response.completed"), + "transport": ("do request failed", "unexpected eof", "connection reset", "stream read error", "stream closed", "internal_error; received from peer"), + "upstream-api-key-validation": ("failed to validate api key", "api key validation failed"), + "authentication": ("unauthorized", "invalid token", "token revoked", "invalid_grant"), + "generic-upstream-failure": ("upstream request failed", "upstream gateway error"), + } + for cause, values in markers.items(): + if any(marker in lower for marker in values): + return cause + if re.match(r"recovered upstream error \\d+\\b", lower): + return "status-only-upstream-failure" + return "unknown" + +def ops_error_analysis(token, account, log_statuses): + account_id = account.get("id") + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + path = f"/api/v1/admin/ops/upstream-errors?account_id={account_id}&view=all&page=1&page_size=500&start_time={start_time}" + response = curl_api("GET", path, bearer=token) + try: + data = data_of(response, "list account ops errors") + except Exception as exc: + return { + "source": "ops-upstream-errors", + "status": "unavailable", + "logObservedStatusCodes": log_statuses, + "reason": compact_error(str(exc)), + "assessment": "insufficient-evidence", + } + rows = items_of(data) + total = data.get("total", len(rows)) if isinstance(data, dict) else len(rows) + detail_limit = 24 if PAYLOAD.get("full") is True else 8 + analyzed = [] + for row in rows: + message = compact_error(row.get("message") if isinstance(row, dict) else "") + analyzed.append({"row": row, "evidence": message, "cause": ops_error_cause(message)}) + for entry in analyzed[:detail_limit]: + row = entry["row"] + error_id = row.get("id") if isinstance(row, dict) else None + if not isinstance(error_id, int): + continue + detail_response = curl_api("GET", f"/api/v1/admin/ops/upstream-errors/{error_id}", bearer=token) + try: + detail = data_of(detail_response, "get account 500 ops error") + except Exception: + continue + fields = [] + for key in ("message", "upstream_error_message", "upstream_error_detail", "error_body", "upstream_errors"): + value = detail.get(key) if isinstance(detail, dict) else None + if isinstance(value, str) and value.strip(): + fields.append(value) + evidence = compact_error(" ".join(fields)) + entry["evidence"] = evidence + entry["cause"] = ops_error_cause(evidence) + + causes = {} + statuses = {} + messages = {} + models = {} + hours = {} + stream_modes = {"stream": 0, "synchronous": 0, "unknown": 0} + concurrency_count = 0 + for entry in analyzed: + row = entry["row"] + error_id = row.get("id") if isinstance(row, dict) else None + evidence = entry["evidence"] + cause = entry["cause"] + causes[cause] = causes.get(cause, 0) + 1 + list_message = compact_error(row.get("message") if isinstance(row, dict) else "") + if list_message: + messages[list_message] = messages.get(list_message, 0) + 1 + status_code = row.get("status_code") if isinstance(row, dict) else None + if isinstance(status_code, int): + statuses[str(status_code)] = statuses.get(str(status_code), 0) + 1 + if cause == "concurrency": + concurrency_count += 1 + model = str(row.get("model") or "unknown") if isinstance(row, dict) else "unknown" + models[model] = models.get(model, 0) + 1 + created_at = str(row.get("created_at") or "") if isinstance(row, dict) else "" + hour = created_at[:13] + ":00" if len(created_at) >= 13 else "unknown" + hours[hour] = hours.get(hour, 0) + 1 + stream = row.get("stream") if isinstance(row, dict) else None + stream_key = "stream" if stream is True else "synchronous" if stream is False else "unknown" + stream_modes[stream_key] += 1 + sample = { + "errorId": error_id, + "createdAt": row.get("created_at") if isinstance(row, dict) else None, + "requestId": row.get("request_id") if isinstance(row, dict) else None, + "model": row.get("model") if isinstance(row, dict) else None, + "stream": row.get("stream") if isinstance(row, dict) else None, + "cause": cause, + "evidence": evidence, + } + entry["sample"] = sample + analyzed_count = len(analyzed) + if analyzed_count == 0: + assessment = "insufficient-evidence" + elif concurrency_count == analyzed_count: + assessment = "concurrency-related" + elif concurrency_count > 0: + assessment = "mixed" + else: + assessment = "no-direct-concurrency-evidence" + dominant_cause = None + dominant_count = 0 + if causes: + dominant_cause, dominant_count = max(causes.items(), key=lambda pair: pair[1]) + dominant_share = round(dominant_count / analyzed_count, 4) if analyzed_count > 0 else None + if dominant_cause == "upstream-api-key-validation": + finding = "The upstream gateway returned HTTP 500 while validating its inbound API key; investigate its key lookup and storage dependencies first." + elif dominant_cause == "concurrency": + finding = "Direct concurrency-limit messages dominate the observed Ops records." + elif dominant_cause is not None: + finding = "The dominant observed Ops cause is " + dominant_cause + "." + else: + finding = "No structured HTTP 500 Ops records were available for cause analysis." + if concurrency_count > 0: + concurrency_action = "consider-lower-limit-after-controlled-comparison" + else: + concurrency_action = "do-not-lower-limit-as-primary-fix" + ranked_messages = sorted(messages.items(), key=lambda pair: (-pair[1], pair[0])) + message_limit = 20 if PAYLOAD.get("full") is True else 5 + return { + "source": "ops-upstream-errors", + "status": "available", + "logObservedStatusCodes": log_statuses, + "opsRecordCount": total, + "analyzedRecordCount": analyzed_count, + "detailRecordCount": min(detail_limit, analyzed_count), + "recordsTruncated": total > analyzed_count, + "statusBuckets": [{"statusCode": int(code), "count": count} for code, count in sorted(statuses.items(), key=lambda pair: int(pair[0]))], + "causeBuckets": [{"cause": cause, "count": count} for cause, count in sorted(causes.items())], + "messageBuckets": [{"message": message, "count": count} for message, count in ranked_messages[:message_limit]], + "concurrencyRelatedRecordCount": concurrency_count, + "assessment": assessment, + "rootCause": { + "dominantCause": dominant_cause, + "dominantCount": dominant_count, + "dominantShare": dominant_share, + "finding": finding, + }, + "errorRateEvidence": { + "errorRecordCount": total, + "requestAttemptDenominator": None, + "errorRate": None, + "reason": "The available Ops endpoint lists errors but does not provide all routing attempts for this account.", + }, + "concurrencyRecommendation": { + "action": concurrency_action, + "reason": "No error-time concurrency history is available; only explicit concurrency error text is treated as direct evidence.", + }, + "runtimeConcurrency": { + "limit": account.get("concurrency"), + "current": account.get("current_concurrency"), + "loadFactor": account.get("load_factor"), + "historicalAtErrorAvailable": False, + }, + **({ + "modelBuckets": [{"model": model, "count": count} for model, count in sorted(models.items(), key=lambda pair: (-pair[1], pair[0]))], + "hourBuckets": [{"hour": hour, "count": count} for hour, count in sorted(hours.items())], + "streamBuckets": [{"mode": mode, "count": count} for mode, count in stream_modes.items() if count > 0], + "samples": [entry["sample"] for entry in analyzed[:detail_limit]], + } if PAYLOAD.get("full") is True else { + "samples": [entry["sample"] for entry in analyzed[:1]], + }), + "disclosure": "Ops records are independently sampled evidence and are not positionally joined to account_upstream_error log lines.", + } + +def gateway_request_path(value): + path = str(value or "").split("?", 1)[0].rstrip("/") + return path in ( + "/responses", "/v1/responses", "/responses/compact", "/v1/responses/compact", + "/messages", "/v1/messages", "/chat/completions", "/v1/chat/completions", + ) + +def native_ops_snapshot(token, group_id, platform=None): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + base_query = f"group_id={group_id}&start_time={start_time}{platform_query}" + + def optional(path, label): + response = curl_api("GET", path, bearer=token) + try: + return {"status": "available", "data": data_of(response, label)} + except Exception as exc: + return {"status": "unavailable", "reason": compact_error(str(exc))} + + overview_result = optional(f"/api/v1/admin/ops/dashboard/overview?{base_query}&query_mode=raw", "get ops dashboard overview") + availability_result = optional(f"/api/v1/admin/ops/account-availability?group_id={group_id}{platform_query}", "get ops account availability") + concurrency_result = optional(f"/api/v1/admin/ops/concurrency?group_id={group_id}{platform_query}", "get ops concurrency") + sink_result = optional("/api/v1/admin/ops/system-logs/health", "get ops system log health") + + overview = overview_result.get("data") if overview_result.get("status") == "available" else None + overview_summary = None + if isinstance(overview, dict): + overview_summary = { + key: overview.get(key) + for key in ( + "start_time", "end_time", "health_score", "success_count", "error_count_total", + "business_limited_count", "error_count_sla", "request_count_total", "request_count_sla", + "error_rate", "upstream_error_rate", "upstream_error_count_excl_429_529", + "upstream_429_count", "upstream_529_count", + ) + } + + availability = availability_result.get("data") if availability_result.get("status") == "available" else None + availability_summary = None + if isinstance(availability, dict): + group_values = availability.get("group") if isinstance(availability.get("group"), dict) else {} + account_values = availability.get("account") if isinstance(availability.get("account"), dict) else {} + selected_group = group_values.get(str(group_id)) or group_values.get(group_id) + unavailable = [] + for value in account_values.values(): + if not isinstance(value, dict) or value.get("group_id") != group_id or value.get("is_available") is True: + continue + unavailable.append({ + "accountId": value.get("account_id"), + "accountName": value.get("account_name"), + "status": value.get("status"), + "rateLimited": value.get("is_rate_limited"), + "rateLimitRemainingSeconds": value.get("rate_limit_remaining_sec"), + "overloaded": value.get("is_overloaded"), + "overloadRemainingSeconds": value.get("overload_remaining_sec"), + "hasError": value.get("has_error"), + "error": compact_error(value.get("error_message")), + }) + availability_summary = { + "enabled": availability.get("enabled"), + "timestamp": availability.get("timestamp"), + "group": selected_group, + "unavailableAccounts": unavailable[:20 if PAYLOAD.get("full") is True else 8], + "unavailableAccountCount": max( + len(unavailable), + max(0, selected_group.get("total_accounts") - selected_group.get("available_count")) + if isinstance(selected_group, dict) + and isinstance(selected_group.get("total_accounts"), int) + and isinstance(selected_group.get("available_count"), int) + else 0, + ), + "unavailableAccountDetailsComplete": ( + not isinstance(selected_group, dict) + or not isinstance(selected_group.get("total_accounts"), int) + or not isinstance(selected_group.get("available_count"), int) + or len(unavailable) >= max(0, selected_group.get("total_accounts") - selected_group.get("available_count")) + ), + } + + concurrency = concurrency_result.get("data") if concurrency_result.get("status") == "available" else None + concurrency_summary = None + if isinstance(concurrency, dict): + group_values = concurrency.get("group") if isinstance(concurrency.get("group"), dict) else {} + account_values = concurrency.get("account") if isinstance(concurrency.get("account"), dict) else {} + selected_group = group_values.get(str(group_id)) or group_values.get(group_id) + loaded = [] + for value in account_values.values(): + if not isinstance(value, dict) or value.get("group_id") != group_id: + continue + if (value.get("current_in_use") or 0) <= 0 and (value.get("waiting_in_queue") or 0) <= 0: + continue + loaded.append({ + "accountId": value.get("account_id"), + "accountName": value.get("account_name"), + "currentInUse": value.get("current_in_use"), + "maxCapacity": value.get("max_capacity"), + "loadPercentage": value.get("load_percentage"), + "waitingInQueue": value.get("waiting_in_queue"), + }) + concurrency_summary = { + "enabled": concurrency.get("enabled"), + "timestamp": concurrency.get("timestamp"), + "group": selected_group, + "activeAccounts": sorted(loaded, key=lambda item: (-(item.get("loadPercentage") or 0), str(item.get("accountName") or "")))[:20 if PAYLOAD.get("full") is True else 8], + } + + sink = sink_result.get("data") if sink_result.get("status") == "available" else None + return { + "source": "sub2api-native-admin-ops", + "overview": {"status": overview_result.get("status"), "summary": overview_summary, "reason": overview_result.get("reason")}, + "accountAvailability": {"status": availability_result.get("status"), "summary": availability_summary, "reason": availability_result.get("reason")}, + "concurrency": {"status": concurrency_result.get("status"), "summary": concurrency_summary, "reason": concurrency_result.get("reason")}, + "systemLogSink": {"status": sink_result.get("status"), "health": sink if isinstance(sink, dict) else None, "reason": sink_result.get("reason")}, + } + +def all_group_error_overview(token, groups): + page_size = 20 + cursor = int(PAYLOAD.get("pageToken") or 0) + ordered = sorted( + [item for item in groups if isinstance(item, dict) and isinstance(item.get("id"), int)], + key=lambda item: item["id"], + ) + remaining = [item for item in ordered if item["id"] > cursor] + page = remaining[:page_size] + summaries = [] + totals = { + "requestCount": 0, + "errorCount": 0, + "businessLimitedCount": 0, + "upstreamErrorCount": 0, + "groupsNeedingAttention": 0, + } + for group in page: + group_id = group["id"] + platform = PAYLOAD.get("platform") or group.get("platform") + snapshot = native_ops_snapshot(token, group_id, platform) + overview = ((snapshot.get("overview") or {}).get("summary") or {}) + availability = ((snapshot.get("accountAvailability") or {}).get("summary") or {}) + availability_group = availability.get("group") if isinstance(availability.get("group"), dict) else {} + concurrency = ((snapshot.get("concurrency") or {}).get("summary") or {}) + concurrency_group = concurrency.get("group") if isinstance(concurrency.get("group"), dict) else {} + request_count = overview.get("request_count_total") + error_count = overview.get("error_count_total") + business_limited = overview.get("business_limited_count") + upstream_error_count = sum( + value for value in ( + overview.get("upstream_error_count_excl_429_529"), + overview.get("upstream_429_count"), + overview.get("upstream_529_count"), + ) if isinstance(value, int) + ) + unavailable_count = availability.get("unavailableAccountCount") + needs_attention = ( + (isinstance(error_count, int) and error_count > 0) + or upstream_error_count > 0 + or (isinstance(unavailable_count, int) and unavailable_count > 0) + ) + totals["requestCount"] += request_count if isinstance(request_count, int) else 0 + totals["errorCount"] += error_count if isinstance(error_count, int) else 0 + totals["businessLimitedCount"] += business_limited if isinstance(business_limited, int) else 0 + totals["upstreamErrorCount"] += upstream_error_count + totals["groupsNeedingAttention"] += 1 if needs_attention else 0 + summaries.append({ + "groupId": group_id, + "groupName": group.get("name"), + "platform": platform, + "status": group.get("status"), + "requestCount": request_count, + "errorCount": error_count, + "errorRate": overview.get("error_rate"), + "upstreamErrorCount": upstream_error_count, + "upstreamErrorRate": overview.get("upstream_error_rate"), + "businessLimitedCount": business_limited, + "totalAccounts": availability_group.get("total_accounts"), + "availableCount": availability_group.get("available_count"), + "rateLimitCount": availability_group.get("rate_limit_count"), + "errorAccountCount": availability_group.get("error_count"), + "unavailableAccountCount": unavailable_count, + "currentInUse": concurrency_group.get("current_in_use"), + "maxCapacity": concurrency_group.get("max_capacity"), + "waitingInQueue": concurrency_group.get("waiting_in_queue"), + "needsAttention": needs_attention, + "opsStatus": { + "overview": (snapshot.get("overview") or {}).get("status"), + "accountAvailability": (snapshot.get("accountAvailability") or {}).get("status"), + "concurrency": (snapshot.get("concurrency") or {}).get("status"), + }, + }) + next_token = str(page[-1]["id"]) if len(remaining) > page_size and page else None + totals["errorRate"] = round(totals["errorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None + totals["upstreamErrorRate"] = round(totals["upstreamErrorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None + return { + "scope": "all-groups-overview", + "window": {"since": PAYLOAD.get("since")}, + "platform": PAYLOAD.get("platform"), + "totalGroups": len(ordered), + "returnedGroups": len(summaries), + "pageSize": page_size, + "pageToken": str(cursor) if cursor > 0 else None, + "nextPageToken": next_token, + "pageTotals": totals, + "groups": summaries, + "disclosure": "Page totals cover only returned groups. Follow nextPageToken for remaining groups and use --group for account root causes and request indexes.", + } + +def native_system_log_lines(token, platform=None): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + markers = ( + "account_temp_unschedulable", "openai.upstream_failover_switching", + "openai.account_select_failed", "openai.forward_failed", "http request completed", + ) + rows_by_id = {} + queries = [] + for marker in markers: + path = f"/api/v1/admin/ops/system-logs?page=1&page_size=200&start_time={start_time}&q={quote(marker, safe='')}{platform_query}" + response = curl_api("GET", path, bearer=token) + try: + data = data_of(response, "list ops system logs") + except Exception as exc: + return None, { + "status": "unavailable", + "source": "sub2api-native-admin-ops-system-logs", + "reason": compact_error(str(exc)), + "fallback": "runtime-container-logs", + } + rows = items_of(data) + total = data.get("total", len(rows)) if isinstance(data, dict) else len(rows) + queries.append({"marker": marker, "returned": len(rows), "total": total, "truncated": total > len(rows)}) + for row in rows: + if not isinstance(row, dict): + continue + row_id = row.get("id") + key = str(row_id) if row_id is not None else json.dumps(row, sort_keys=True, default=str) + rows_by_id[key] = row + lines = [] + for row in sorted(rows_by_id.values(), key=lambda item: str(item.get("created_at") or "")): + extra = dict(row.get("extra") or {}) if isinstance(row.get("extra"), dict) else {} + for source_key, target_key in ( + ("request_id", "request_id"), ("client_request_id", "client_request_id"), + ("account_id", "account_id"), ("platform", "platform"), ("model", "model"), + ): + if extra.get(target_key) is None and row.get(source_key) is not None: + extra[target_key] = row.get(source_key) + message = str(row.get("message") or "") + lines.append(str(row.get("created_at") or "") + "\\t" + message + "\\t" + json.dumps(extra, separators=(",", ":"), default=str)) + return lines, { + "status": "available", + "source": "sub2api-native-admin-ops-system-logs", + "uniqueEventCount": len(lines), + "queries": queries, + "fallbackUsed": False, + } + +def policy_effectiveness(lines, accounts, source, group_id=None): + account_names = {str(item.get("id")): item.get("name") for item in accounts} + account_ids = set(account_names.keys()) + requests = {} + counts = { + "tempUnschedulableEvents": 0, + "failoverEvents": 0, + "selectFailedEvents": 0, + "forwardFailedEvents": 0, + "gatewayCompletedEvents": 0, + "gatewayFinalErrorEvents": 0, + } + missing_request_ids = { + "tempUnschedulable": 0, + "failover": 0, + "selectFailed": 0, + "forwardFailed": 0, + "gatewayCompleted": 0, + } + temp_accounts = {} + temp_samples = [] + failover_samples = [] + select_failed_samples = [] + final_statuses = {} + event_line_count = 0 + + def request_entry(request_id): + if not isinstance(request_id, str) or not request_id: + return None + if request_id not in requests: + requests[request_id] = { + "requestId": request_id, + "tempUnschedulableCount": 0, + "failoverCount": 0, + "selectFailedCount": 0, + "forwardFailedCount": 0, + "finalStatus": None, + } + return requests[request_id] + + for line in lines: + if not any(marker in line for marker in ( + "account_temp_unschedulable", "openai.upstream_failover_switching", + "openai.account_select_failed", "openai.forward_failed", "http request completed", + )): + continue + json_start = line.find("{") + if json_start < 0: + continue + try: + item = json.loads(line[json_start:]) + except json.JSONDecodeError: + continue + if not isinstance(item, dict): + continue + at = line.split("\\t", 1)[0].strip() or None + request_id = item.get("request_id") + account_id = item.get("account_id") + event_group_id = item.get("group_id") + if event_group_id is not None and group_id is not None and str(event_group_id) != str(group_id): + continue + if account_id is not None and str(account_id) not in account_ids: + continue + explicitly_scoped = account_id is not None or event_group_id is not None + if not explicitly_scoped and (not isinstance(request_id, str) or request_id not in requests): + continue + request = request_entry(request_id) + base = { + "at": at, + "requestId": request_id, + "accountId": account_id, + "accountName": account_names.get(str(account_id)), + } + if "account_temp_unschedulable" in line: + event_line_count += 1 + counts["tempUnschedulableEvents"] += 1 + if request is not None: + request["tempUnschedulableCount"] += 1 + else: + missing_request_ids["tempUnschedulable"] += 1 + key = str(account_id) if account_id is not None else "unknown" + if key not in temp_accounts: + temp_accounts[key] = { + "accountId": account_id, + "accountName": account_names.get(str(account_id)), + "count": 0, + } + temp_accounts[key]["count"] += 1 + if len(temp_samples) < 20: + temp_samples.append({ + **base, + "statusCode": item.get("status_code"), + "ruleIndex": item.get("rule_index"), + "matchedKeyword": item.get("matched_keyword"), + "until": item.get("until") or item.get("temp_unschedulable_until"), + "reason": compact_error(item.get("reason") or item.get("error")), + }) + elif "openai.upstream_failover_switching" in line: + event_line_count += 1 + counts["failoverEvents"] += 1 + if request is not None: + request["failoverCount"] += 1 + else: + missing_request_ids["failover"] += 1 + if len(failover_samples) < 20: + failover_samples.append({ + **base, + "switchCount": item.get("switch_count"), + "maxSwitches": item.get("max_switches"), + }) + elif "openai.account_select_failed" in line: + event_line_count += 1 + counts["selectFailedEvents"] += 1 + if request is not None: + request["selectFailedCount"] += 1 + else: + missing_request_ids["selectFailed"] += 1 + if len(select_failed_samples) < 20: + select_failed_samples.append({ + **base, + "excludedAccountCount": item.get("excluded_account_count"), + "error": compact_error(item.get("error")), + }) + elif "openai.forward_failed" in line: + event_line_count += 1 + counts["forwardFailedEvents"] += 1 + if request is not None: + request["forwardFailedCount"] += 1 + else: + missing_request_ids["forwardFailed"] += 1 + elif "http request completed" in line and gateway_request_path(item.get("path")): + event_line_count += 1 + counts["gatewayCompletedEvents"] += 1 + status_code = item.get("status_code") + if request is not None and isinstance(status_code, int): + request["finalStatus"] = status_code + elif request is None: + missing_request_ids["gatewayCompleted"] += 1 + if isinstance(status_code, int): + key = str(status_code) + final_statuses[key] = final_statuses.get(key, 0) + 1 + if status_code >= 400: + counts["gatewayFinalErrorEvents"] += 1 + + correlated = [item for item in requests.values() if item["failoverCount"] > 0] + succeeded = [item for item in correlated if isinstance(item.get("finalStatus"), int) and item["finalStatus"] < 400] + failed = [item for item in correlated if isinstance(item.get("finalStatus"), int) and item["finalStatus"] >= 400] + incomplete = [item for item in correlated if not isinstance(item.get("finalStatus"), int)] + cooled_then_failover = [item for item in correlated if item["tempUnschedulableCount"] > 0] + forward_with_http_success = [ + item for item in requests.values() + if item["forwardFailedCount"] > 0 and isinstance(item.get("finalStatus"), int) and item["finalStatus"] < 400 + ] + forward_with_http_error = [ + item for item in requests.values() + if item["forwardFailedCount"] > 0 and isinstance(item.get("finalStatus"), int) and item["finalStatus"] >= 400 + ] + forward_outcome_missing = [ + item for item in requests.values() + if item["forwardFailedCount"] > 0 and not isinstance(item.get("finalStatus"), int) + ] + sample_limit = 20 if PAYLOAD.get("full") is True else (2 if PAYLOAD.get("account") is not None else 5) + completed = counts["gatewayCompletedEvents"] + final_errors = counts["gatewayFinalErrorEvents"] + result = { + "source": source, + "scope": "group-membership-window", + "attribution": "Events with group_id are exact. Events without group_id are attributed by account membership and request_id correlation; shared-account events can appear in more than one group drilldown.", + "eventLineCount": event_line_count, + **counts, + "tempUnschedulableAccounts": sorted(temp_accounts.values(), key=lambda item: (-item["count"], str(item.get("accountName") or ""))), + "failoverRequestCount": len(correlated), + "failoverSucceededRequestCount": len(succeeded), + "failoverFailedRequestCount": len(failed), + "failoverOutcomeMissingRequestCount": len(incomplete), + "cooledThenFailoverRequestCount": len(cooled_then_failover), + "forwardFailureWithHttpSuccessRequestCount": len(forward_with_http_success), + "forwardFailureWithHttpErrorRequestCount": len(forward_with_http_error), + "forwardFailureOutcomeMissingRequestCount": len(forward_outcome_missing), + "requestIdCoverage": { + "missingByEventType": missing_request_ids, + "tempToFailoverCorrelationAvailable": counts["tempUnschedulableEvents"] > missing_request_ids["tempUnschedulable"], + }, + "gatewayFinalStatusBuckets": [ + {"statusCode": int(code), "count": count} + for code, count in sorted(final_statuses.items(), key=lambda pair: int(pair[0])) + ], + "gatewayOutcomeEvidence": { + "completedLogEventCount": completed, + "finalErrorLogEventCount": final_errors, + "observedFinalErrorShare": round(final_errors / completed, 4) if completed > 0 else None, + "denominator": "gateway http request completed log events for Responses, Messages, and Chat Completions paths", + "coverage": "since-window-and-tail-bounded", + "completeTrafficDenominatorGuaranteed": False, + }, + "requestDrilldown": { + "succeededAfterFailover": [item["requestId"] for item in succeeded[:sample_limit]], + "failedAfterFailover": [item["requestId"] for item in failed[:sample_limit]], + "outcomeMissingAfterFailover": [item["requestId"] for item in incomplete[:sample_limit]], + "selectFailed": [item["requestId"] for item in requests.values() if item["selectFailedCount"] > 0][:sample_limit], + "forwardFailureWithHttpSuccess": [item["requestId"] for item in forward_with_http_success[:sample_limit]], + "forwardFailureWithHttpError": [item["requestId"] for item in forward_with_http_error[:sample_limit]], + "forwardFailureOutcomeMissing": [item["requestId"] for item in forward_outcome_missing[:sample_limit]], + }, + "assessment": ( + "failover-observed-with-success" if succeeded else + "failover-observed-without-success" if correlated else + "no-failover-observed" + ), + "disclosure": "Rule matches require account_temp_unschedulable evidence; status codes alone are not treated as rule hits. Request outcomes are joined only by request_id. HTTP success can coexist with forward_failed on streaming responses and does not prove the client received a complete terminal event.", + } + if PAYLOAD.get("full") is True: + result["samples"] = { + "tempUnschedulable": temp_samples, + "failovers": failover_samples, + "selectFailures": select_failed_samples, + } + return result + +def observed_runtime_errors(token, accounts, group_id, platform=None): selector = PAYLOAD.get("account") selected = [item for item in accounts if selector is None or str(item.get("id")) == str(selector) or item.get("name") == selector] if selector is not None and len(selected) != 1: raise RuntimeError("account-not-found-or-ambiguous: " + str(selector)) - selected_ids = set(item.get("id") for item in selected) + selected_ids = set(str(item.get("id")) for item in selected) stats = {} for item in selected: - stats[item.get("id")] = { + stats[str(item.get("id"))] = { "accountName": item.get("name"), "accountId": item.get("id"), "upstreamEventCount": 0, @@ -430,8 +1438,50 @@ def observed_runtime_errors(accounts): "lastSeenAt": None, "errorSamples": [], } + native_ops = native_ops_snapshot(token, group_id, platform) + lines, system_log_source = native_system_log_lines(token, platform) + if lines is None: + lines = runtime_log_lines() + system_log_source["fallbackUsed"] = True + else: + marker_queries = [item for item in system_log_source.get("queries", []) if item.get("marker") != "http request completed"] + marker_total = sum(item.get("total", 0) for item in marker_queries if isinstance(item.get("total"), int)) + overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) + upstream_count = overview_summary.get("upstream_error_count_excl_429_529") or 0 + upstream_count += overview_summary.get("upstream_429_count") or 0 + upstream_count += overview_summary.get("upstream_529_count") or 0 + if marker_total == 0 and upstream_count > 0: + runtime_event_lines = [ + line for line in runtime_log_lines() + if any(marker in line for marker in ( + "account_temp_unschedulable", "openai.upstream_failover_switching", + "openai.account_select_failed", "openai.forward_failed", + )) + ] + lines.extend(runtime_event_lines) + system_log_source["source"] = "sub2api-native-admin-ops-with-runtime-event-fallback" + system_log_source["fallbackUsed"] = True + system_log_source["fallbackReason"] = "native system-log index returned no policy events while the native overview reported upstream errors" + system_log_source["fallbackEventCount"] = len(runtime_event_lines) + effectiveness = policy_effectiveness(lines, accounts, system_log_source.get("source"), group_id) + effectiveness["sourceStatus"] = system_log_source + overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) + request_total = overview_summary.get("request_count_total") + error_total = overview_summary.get("error_count_total") + if isinstance(request_total, int) and isinstance(error_total, int): + effectiveness["gatewayOutcomeEvidence"] = { + "source": "sub2api-native-admin-ops-dashboard-overview", + "requestCountTotal": request_total, + "errorCountTotal": error_total, + "errorRate": overview_summary.get("error_rate"), + "upstreamErrorRate": overview_summary.get("upstream_error_rate"), + "businessLimitedCount": overview_summary.get("business_limited_count"), + "startTime": overview_summary.get("start_time"), + "endTime": overview_summary.get("end_time"), + "completeTrafficDenominatorGuaranteed": True, + } scanned = 0 - for line in runtime_log_lines(): + for line in lines: if "account_upstream_error" not in line and "openai.forward_failed" not in line: continue json_start = line.find("{") @@ -442,11 +1492,12 @@ def observed_runtime_errors(accounts): except json.JSONDecodeError: continue account_id = item.get("account_id") if isinstance(item, dict) else None - if account_id not in selected_ids: + account_key = str(account_id) + if account_key not in selected_ids: continue scanned += 1 at = line.split("\t", 1)[0].strip() or None - current = stats[account_id] + current = stats[account_key] if at and (current["lastSeenAt"] is None or at > current["lastSeenAt"]): current["lastSeenAt"] = at if "account_upstream_error" in line: @@ -467,7 +1518,7 @@ def observed_runtime_errors(accounts): rows = [] aggregate = {} for item in selected: - current = stats[item.get("id")] + current = stats[str(item.get("id"))] observed = [{"statusCode": int(code), "count": count} for code, count in sorted(current["observedStatusCodes"].items(), key=lambda pair: int(pair[0]))] forwarded = [{"statusCode": int(code), "count": count} for code, count in sorted(current["forwardFailureStatusCodes"].items(), key=lambda pair: int(pair[0]))] for entry in observed: @@ -475,36 +1526,211 @@ def observed_runtime_errors(accounts): aggregate[key] = aggregate.get(key, 0) + entry["count"] current["observedStatusCodes"] = observed current["forwardFailureStatusCodes"] = forwarded + analysis = ops_error_analysis(token, item, observed) + has_ops_errors = analysis.get("status") == "available" and (analysis.get("opsRecordCount") or 0) > 0 + if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or has_ops_errors: + if PAYLOAD.get("full") is not True and analysis.get("status") == "available": + analysis = { + "source": analysis.get("source"), + "status": analysis.get("status"), + "opsRecordCount": analysis.get("opsRecordCount"), + "recordsTruncated": analysis.get("recordsTruncated"), + "statusBuckets": analysis.get("statusBuckets"), + "causeBuckets": analysis.get("causeBuckets"), + "rootCause": analysis.get("rootCause"), + "errorRateEvidence": analysis.get("errorRateEvidence"), + "concurrencyRecommendation": analysis.get("concurrencyRecommendation"), + "runtimeConcurrency": analysis.get("runtimeConcurrency"), + "samples": (analysis.get("samples") or [])[:1], + } + current["opsAnalysis"] = analysis if PAYLOAD.get("full") is not True: current["errorSamples"] = current["errorSamples"][:1] - if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or selector is not None: + if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or has_ops_errors or selector is not None: rows.append(current) + root_causes = {} + account_roots = [] + for row in rows: + analysis = row.get("opsAnalysis") + if not isinstance(analysis, dict) or analysis.get("status") != "available": + continue + for bucket in analysis.get("causeBuckets") or []: + cause = bucket.get("cause") + count = bucket.get("count") + if isinstance(cause, str) and isinstance(count, int): + root_causes[cause] = root_causes.get(cause, 0) + count + root = analysis.get("rootCause") or {} + account_roots.append({ + "accountId": row.get("accountId"), + "accountName": row.get("accountName"), + "dominantCause": root.get("dominantCause"), + "dominantCount": root.get("dominantCount"), + "dominantShare": root.get("dominantShare"), + }) + ranked_causes = sorted(root_causes.items(), key=lambda pair: (-pair[1], pair[0])) + native_overview = native_ops.get("overview") or {} + native_overview_values = native_overview.get("summary") if isinstance(native_overview.get("summary"), dict) else native_overview + native_error_rate_available = isinstance(native_overview_values.get("error_rate") if isinstance(native_overview_values, dict) else None, (int, float)) + account_count_with_errors = sum( + 1 for item in rows + if item["upstreamEventCount"] > 0 + or item["forwardFailureCount"] > 0 + or (((item.get("opsAnalysis") or {}).get("opsRecordCount") or 0) > 0) + ) + if selector is None and PAYLOAD.get("full") is not True: + for row in rows: + row.pop("opsAnalysis", None) + if PAYLOAD.get("full") is not True: + overview = ((native_ops.get("overview") or {}).get("summary") or {}) + availability = ((native_ops.get("accountAvailability") or {}).get("summary") or {}) + concurrency = ((native_ops.get("concurrency") or {}).get("summary") or {}) + sink = ((native_ops.get("systemLogSink") or {}).get("health") or {}) + unavailable_accounts = availability.get("unavailableAccounts") or [] + if selector is not None: + unavailable_accounts = [item for item in unavailable_accounts if item.get("accountId") in selected_ids] + native_ops = { + "source": native_ops.get("source"), + "overview": { + "status": (native_ops.get("overview") or {}).get("status"), + "healthScore": overview.get("health_score"), + "requestCount": overview.get("request_count_total"), + "errorCount": overview.get("error_count_total"), + "errorRate": overview.get("error_rate"), + "upstreamErrorRate": overview.get("upstream_error_rate"), + "businessLimitedCount": overview.get("business_limited_count"), + }, + "accountAvailability": { + "status": (native_ops.get("accountAvailability") or {}).get("status"), + "group": availability.get("group"), + "unavailableAccountCount": availability.get("unavailableAccountCount"), + "unavailableAccounts": unavailable_accounts, + }, + "concurrency": { + "status": (native_ops.get("concurrency") or {}).get("status"), + "group": concurrency.get("group"), + }, + "systemLogSink": { + "status": (native_ops.get("systemLogSink") or {}).get("status"), + "queueDepth": sink.get("queue_depth"), + "droppedCount": sink.get("dropped_count"), + "writeFailedCount": sink.get("write_failed_count"), + }, + } + source_status = effectiveness.get("sourceStatus") or {} + compact_source_status = { + "status": source_status.get("status"), + "source": source_status.get("source"), + "fallbackUsed": source_status.get("fallbackUsed"), + "fallbackReason": source_status.get("fallbackReason"), + "fallbackEventCount": source_status.get("fallbackEventCount"), + "nativeIndexedEventCounts": { + item.get("marker"): item.get("total") + for item in source_status.get("queries", []) + if isinstance(item, dict) + }, + } + drilldown = effectiveness.get("requestDrilldown") or {} + effectiveness = { + "source": effectiveness.get("source"), + "scope": effectiveness.get("scope"), + "attribution": effectiveness.get("attribution"), + "assessment": effectiveness.get("assessment"), + "ruleMatches": { + "tempUnschedulableEvents": effectiveness.get("tempUnschedulableEvents"), + "accounts": effectiveness.get("tempUnschedulableAccounts"), + }, + "failover": { + "events": effectiveness.get("failoverEvents"), + "requests": effectiveness.get("failoverRequestCount"), + "succeeded": effectiveness.get("failoverSucceededRequestCount"), + "failed": effectiveness.get("failoverFailedRequestCount"), + "outcomeMissing": effectiveness.get("failoverOutcomeMissingRequestCount"), + }, + "forwardFailure": { + "events": effectiveness.get("forwardFailedEvents"), + "withHttpSuccess": effectiveness.get("forwardFailureWithHttpSuccessRequestCount"), + "withHttpError": effectiveness.get("forwardFailureWithHttpErrorRequestCount"), + "outcomeMissing": effectiveness.get("forwardFailureOutcomeMissingRequestCount"), + }, + "selectFailedEvents": effectiveness.get("selectFailedEvents"), + "requestIdCoverage": effectiveness.get("requestIdCoverage"), + "gatewayOutcomeEvidence": effectiveness.get("gatewayOutcomeEvidence"), + "requestDrilldown": {key: value for key, value in drilldown.items() if value}, + "sourceStatus": compact_source_status, + "disclosure": effectiveness.get("disclosure"), + } + if selector is not None: + native_ops["accountAvailability"].pop("group", None) + native_ops.pop("concurrency", None) + native_ops.pop("systemLogSink", None) + effectiveness["sourceStatus"] = { + "source": compact_source_status.get("source"), + "fallbackUsed": compact_source_status.get("fallbackUsed"), + "fallbackReason": compact_source_status.get("fallbackReason"), + } return { "window": {"since": PAYLOAD.get("since"), "tail": PAYLOAD.get("tail"), "matchedLogEvents": scanned}, "aggregateObservedStatusCodes": [{"statusCode": int(code), "count": count} for code, count in sorted(aggregate.items(), key=lambda pair: int(pair[0]))], - "accountCountWithErrors": sum(1 for item in rows if item["upstreamEventCount"] > 0 or item["forwardFailureCount"] > 0), - "accounts": rows, + "accountCountWithErrors": account_count_with_errors, + "rootCauseAnalysis": { + "source": "ops-upstream-errors", + "scope": "account-window-for-group-members", + "attribution": "The upstream-errors endpoint is account scoped. If accounts belong to multiple groups, these cause buckets are shared evidence and must not be added across groups.", + "causeBuckets": [{"cause": cause, "count": count} for cause, count in ranked_causes], + "dominantCause": ranked_causes[0][0] if ranked_causes else None, + "dominantCount": ranked_causes[0][1] if ranked_causes else 0, + "accountRoots": account_roots, + "errorRateAvailable": native_error_rate_available, + }, + "nativeOps": native_ops, + "policyEffectiveness": effectiveness, + **( + {"accounts": rows} + if selector is not None or PAYLOAD.get("full") is True + else {"accountDrilldown": "rerun runtime errors with --account for per-account evidence"} + ), "message": "observedStatusCodes comes only from account_upstream_error; forward failures are supplemental message evidence", } def runtime_result(): token = login() - group = next((item for item in list_groups(token) if item.get("name") == PAYLOAD["groupName"]), None) + action = PAYLOAD["action"] + if action == "errors" and PAYLOAD.get("allGroups") is True: + groups = list_groups(token, PAYLOAD.get("platform")) + overview = all_group_error_overview(token, groups) + return { + "ok": True, + "operation": "errors", + "mutation": False, + "endpoint": "host-docker" if RUNTIME_MODE == "host-docker" else "k3s-service", + "allGroups": overview, + "valuesPrinted": False, + } + group_selector = PAYLOAD.get("group") + group_platform = PAYLOAD.get("platform") if action == "errors" else "openai" + if action == "errors" and group_selector is None: + group_platform = "openai" + groups = list_groups(token, group_platform) + selector = str(group_selector if group_selector is not None else PAYLOAD["groupName"]) + matching_groups = [item for item in groups if str(item.get("id")) == selector or item.get("name") == selector] + if len(matching_groups) > 1: + raise RuntimeError("group-selector-ambiguous: " + selector) + group = matching_groups[0] if matching_groups else None if not isinstance(group, dict) or group.get("id") is None: - raise RuntimeError("pool-group-not-found") - accounts = list_group_accounts(token, group["id"]) + raise RuntimeError("group-not-found: " + selector) + effective_platform = group_platform or group.get("platform") + accounts = list_group_accounts(token, group["id"], effective_platform) base = { - "group": {"id": group.get("id"), "name": group.get("name"), "status": group.get("status")}, + "group": {"id": group.get("id"), "name": group.get("name"), "platform": effective_platform, "status": group.get("status")}, "templates": [template_summary(item) for item in PAYLOAD["templates"]], "endpoint": "host-docker" if RUNTIME_MODE == "host-docker" else "k3s-service", "valuesPrinted": False, } - action = PAYLOAD["action"] if action == "list": details = [account_detail(token, item["id"]) for item in accounts] return {**base, "ok": True, "operation": "list", "mutation": False, "accountCount": len(details), "accounts": [account_summary(item) for item in sorted(details, key=lambda value: value.get("name") or "")]} if action == "errors": - errors = observed_runtime_errors(accounts) + errors = observed_runtime_errors(token, accounts, group["id"], effective_platform) return {**base, "ok": True, "operation": "errors", "mutation": False, "errors": errors} detail = find_account(token, accounts, str(PAYLOAD["account"])) if action == "get": diff --git a/scripts/src/platform-infra-sub2api-codex/types.ts b/scripts/src/platform-infra-sub2api-codex/types.ts index 347d4685..27e24ba1 100644 --- a/scripts/src/platform-infra-sub2api-codex/types.ts +++ b/scripts/src/platform-infra-sub2api-codex/types.ts @@ -348,9 +348,9 @@ export function codexPoolHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601", "bun scripts/cli.ts platform-infra sub2api codex-pool sync [--target D601] --confirm [--prune-removed]", "bun scripts/cli.ts platform-infra sub2api codex-pool validate [--target D601] [--full|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--full|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account [--target PK01] [--full|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--account ] [--since 24h] [--tail 50000] [--target PK01] [--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--json|--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account [--target PK01] [--json|--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--group |--all-groups] [--platform ] [--page-token ] [--account ] [--since 24h] [--tail 50000] [--target PK01] [--json|--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --account --template [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --account --kind temp-unschedulable [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]", From b9ede0471e3f46b8b62ba797d7d31787795c5619 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 06:51:24 +0200 Subject: [PATCH 2/4] feat: expose sub2api timeout timing diagnostics --- .../unidesk-sub2api/references/codex-pool.md | 6 + .../R2.2.5_Task_Report.md | 30 +++ docs/MDTODO/sub2api-upstream-reliability.md | 3 + .../remote-python-sync.ts | 35 ++++ .../platform-infra-sub2api-codex/render.ts | 20 +- .../platform-infra-sub2api-codex/runtime.ts | 176 +++++++++++++++++- 6 files changed, 265 insertions(+), 5 deletions(-) create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.5_Task_Report.md diff --git a/.agents/skills/unidesk-sub2api/references/codex-pool.md b/.agents/skills/unidesk-sub2api/references/codex-pool.md index 26811e27..a9d7fdb5 100644 --- a/.agents/skills/unidesk-sub2api/references/codex-pool.md +++ b/.agents/skills/unidesk-sub2api/references/codex-pool.md @@ -14,6 +14,10 @@ - dashboard 请求量和客户错误率按分组精确统计； - 账号被多个分组共享时，账号级 upstream-errors 和缺少 `group_id` 的 runtime 事件属于共享证据，禁止跨分组相加； - dashboard overview 提供请求分母和错误率； + - dashboard overview 的 `ttft` 提供成功流式请求的首 token 分位数，`duration` 提供成功请求总耗时分位数；二者不能互相替代； + - `runtime errors` 只读披露运行实例实际生效的 OpenAI response-header timeout，并按 Sub2API 的 config file、process env、内置默认值优先级标注来源；通用 response-header timeout 不得解释为 OpenAI timeout； + - timeout 候选表同时展示固定对照值和“TTFT P99 加 15 秒后向上取整到 15 秒”的观测下限；观测下限只用于排除明显过低值，不是可直接下发的安全推荐； + - `/admin/ops/requests` 的总耗时长尾只按原生 `duration_ms` 统计；错误记录没有 `duration_ms` 覆盖时显示不可用，禁止把零记录解释为没有慢失败； - account availability 提供可用、限流、过载和错误状态； - concurrency 提供容量与排队； - upstream errors 提供账号根因； @@ -105,6 +109,8 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 `trace --request-id ` 是只读 request 追溯报表，不触发 probe、不修改账号。命令优先使用原生 `/admin/ops/requests` 获取请求结果，并使用 `/admin/ops/system-logs` 获取 request-id 索引事件；target runtime 日志只补原生索引未保留的邻近事件。报表显式输出 target、runtime mode、主数据源、上下文数据源和读取状态。默认输出请求开始/最终状态、failover、`account_select_failed`、`forward_failed`、窗口内 `account_temp_unschedulable`、admin schedulable 写入计数和当前账号快照。`forward_failed` 与 HTTP 200 并存且没有 failover 时，结果标记为 `degraded`，不能解释为完整流式成功；`reason=failover-attempted-no-candidate` 表示 Sub2API 已进入自动切号，但排除当前失败账号后没有可用候选。需要机器处理时使用 `--raw`，需要原始匹配行时加 `--show-lines`。 +`trace` 的时间线以原生索引事件时间戳为主。原生 request 记录只有完成时间、但最终日志带 `latency_ms` 时，命令用“完成时间减总耗时”重建参考起点，并披露首次 failover、选号失败和最终响应相对耗时。Sub2API Ops 没有客户 deadline 时，客户剩余时间必须显示不可用，不能用 `latency_ms` 推断。 + 对已支持的 k3s target，`sync --confirm`、`validate` 和 sentinel 操作可能超过单次 SSH/runtime 短连接窗口；远端 job/poll 边界见 [sentinel.md](sentinel.md)。 不要给 UniDesk-managed Codex accounts 开 Sub2API `pool_mode`。UniDesk 期望的 failover 是把失败账号临时标记为 unschedulable，让同组其他账号接手；`pool_mode` 会重试同一个 account path。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.5_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.5_Task_Report.md new file mode 100644 index 00000000..853e3657 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.5_Task_Report.md @@ -0,0 +1,30 @@ +# R2.2.5 任务报告 + +## 结果 + +- 已增强 `runtime errors` 的只读分析：默认 Kubernetes 风格表格披露运行实例实际 OpenAI response-header timeout、来源、通用 timeout 是否适用、各分组 TTFT/总耗时分位数、固定候选和跨组观测下限。 +- 实际 timeout 按 Sub2API 的 config file、process env、内置默认值优先级解析，只输出目标字段和来源，不输出 Secret；PK01 当前为 process env `0`，即 OpenAI timeout 禁用，通用 600 秒不适用于 OpenAI。 +- 使用原生 dashboard `ttft`，不再用总耗时冒充 TTFT；错误 `duration_ms` 无覆盖时显示 `-`，不把零误判为没有慢失败。 +- all-groups 概要返回每组 TTFT P99/Duration P99，并按当前分页内最大 TTFT P99 加 15 秒、向上取整到 15 秒生成观测下限；该值只用于排除明显过低候选，不是下发建议。 +- 已增强 `trace` 时间线：利用原生索引时间戳和最终 `latency_ms` 重建请求参考起点，显示首次 failover、选号失败、最终响应和 failover-to-final 的相对耗时；客户 deadline 不存在时明确显示不可用。 + +## 真实只读证据 + +- 24 小时 all-groups：25,334 请求、364 客户错误，客户错误率 1.44%，上游错误率 3.92%。 +- group 2：TTFT P99 285,323ms、最大 857,722ms；30/45/60 秒均低于已成功请求 P99，跨组观测下限为 315 秒。 +- group 3：TTFT P99 168,299ms、最大 1,558,296ms；其单组观测下限为 195 秒。 +- request `cf6b6257-9640-414f-918d-c55180fb6a8d`：总耗时 41,599ms，约 41,598ms 后首次 failover，约 1ms 后 `context canceled` 并最终 502。 +- request `4ee54daa-f41c-4f50-bd72-11080c38e299`：总耗时 730,835ms，约 730,834ms 后首次 failover，约 1ms 后 `context canceled` 并最终 502。 +- 结论：当前穿透错误的直接时间线证据是首次上游等待占用了几乎全部请求生命周期，切号本身并不慢；客户 deadline 未被 Ops 记录，不能计算真正剩余预算。 + +## 验证 + +- `bun --check`：`runtime.ts`、`remote-python-sync.ts`、`render.ts`、`output.ts` 均通过。 +- `git diff --check` 通过。 +- `bun test scripts/src/output.test.ts`：2 pass，0 fail。 +- `runtime.ts` 为 1954 行，保持在 2000 行以下。 +- 已真实回归 24 小时 all-groups、group 2/group 3 详情及两条代表 request trace。 + +## 变更边界 + +本任务只修改 UniDesk CLI 和 skill 参考，没有修改 Sub2API 源码、owning YAML、PK01 环境变量、容器或运行面配置。 diff --git a/docs/MDTODO/sub2api-upstream-reliability.md b/docs/MDTODO/sub2api-upstream-reliability.md index 8d56aa2c..ef547536 100644 --- a/docs/MDTODO/sub2api-upstream-reliability.md +++ b/docs/MDTODO/sub2api-upstream-reliability.md @@ -32,6 +32,9 @@ 重新分析所有 Sub2API 分组最近的用户可见错误，并把 runtime errors 泛化为 all-groups 与精确 group 下钻；默认输出必须提供语义摘要、索引和渐进披露，禁止用手工 limit 规避输出问题，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.4_Task_Report.md)。 +#### R2.2.5 [completed] + +增强 Sub2API CLI 的纯分析能力：只读披露实际 OpenAI response-header timeout、TTFT 分位数、长等待请求和 failover 剩余时间窗，并使用原生 admin/ops 数据分析候选配置，不修改 Sub2API 配置、源码或运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.5_Task_Report.md)。 ### R2.3 调查并收敛 Sub2API 原生 system-log 索引未保留策略事件的能力缺口，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.3_Task_Report.md)。 diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts index 88c75360..47c2374d 100644 --- a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts @@ -3145,6 +3145,25 @@ def run_trace(): "latencyMs": native_request.get("duration_ms"), "source": "sub2api-native-admin-ops-requests", } + reference_at = request_start.get("at") if isinstance(request_start, dict) else None + reference_epoch = log_time_epoch({"_at": reference_at}) if isinstance(reference_at, str) else first_epoch + reference_source = "request-start" if isinstance(reference_at, str) and reference_epoch is not None else "first-matched-event" + native_duration_ms = native_request.get("duration_ms") if isinstance(native_request, dict) else None + if not isinstance(native_duration_ms, int) and isinstance(final_event, dict): + native_duration_ms = final_event.get("latencyMs") + if ( + isinstance(request_start, dict) + and request_start.get("source") == "sub2api-native-admin-ops-requests" + and isinstance(native_duration_ms, int) + and native_duration_ms >= 0 + and reference_epoch is not None + ): + reference_epoch -= native_duration_ms / 1000 + reference_at = datetime.fromtimestamp(reference_epoch, timezone.utc).isoformat().replace("+00:00", "Z") + reference_source = "native-completion-minus-duration" + for event in events: + epoch = log_time_epoch({"_at": event.get("at")}) if isinstance(event.get("at"), str) else None + event["elapsedMs"] = round((epoch - reference_epoch) * 1000) if epoch is not None and reference_epoch is not None else None failovers = [item for item in events if item.get("type") == "failover"] select_failures = [item for item in events if item.get("type") == "select-failed"] upstream_errors = [item for item in events if item.get("type") == "upstream-error"] @@ -3157,6 +3176,11 @@ def run_trace(): window_select_failures = [item for item in window_events if item.get("type") == "select-failed"] window_forward_failures = [item for item in window_events if item.get("type") == "forward-failed"] untried_schedulable_accounts = trace_untried_schedulable_accounts(failovers, final_event or {}, account_snapshot) + first_failover_elapsed = failovers[0].get("elapsedMs") if failovers else None + first_select_failed_elapsed = select_failures[0].get("elapsedMs") if select_failures else None + final_epoch = log_time_epoch({"_at": final_event.get("at")}) if isinstance(final_event, dict) and isinstance(final_event.get("at"), str) else None + final_elapsed = round((final_epoch - reference_epoch) * 1000) if final_epoch is not None and reference_epoch is not None else None + failover_to_final = final_elapsed - first_failover_elapsed if isinstance(final_elapsed, int) and isinstance(first_failover_elapsed, int) else None reason = trace_reason(events, final_event) if not matched: outcome = "not-found" @@ -3220,6 +3244,17 @@ def run_trace(): "diagnostics": { "failoverBudgetExhausted": failover_budget_exhausted(failovers, final_event or {}), "untriedSchedulableAccounts": untried_schedulable_accounts, + "timing": { + "referenceAt": reference_at if isinstance(reference_at, str) else (events[0].get("at") if events else None), + "referenceSource": reference_source, + "firstFailoverElapsedMs": first_failover_elapsed, + "firstSelectFailedElapsedMs": first_select_failed_elapsed, + "finalElapsedMs": final_elapsed, + "failoverToFinalMs": failover_to_final, + "clientDeadlineAvailable": False, + "clientDeadlineRemainingMs": None, + "attribution": "Elapsed values come from indexed event timestamps. The client deadline is not present in Sub2API Ops records, so remaining request budget cannot be calculated.", + }, }, "accountSnapshot": account_snapshot, "accountSnapshotError": account_snapshot_error, diff --git a/scripts/src/platform-infra-sub2api-codex/render.ts b/scripts/src/platform-infra-sub2api-codex/render.ts index a7052e38..173d1110 100644 --- a/scripts/src/platform-infra-sub2api-codex/render.ts +++ b/scripts/src/platform-infra-sub2api-codex/render.ts @@ -401,6 +401,8 @@ export function renderTraceReport( const forwardFailures = recordArray(parsed.forwardFailures); const tempUnschedulable = recordArray(parsed.tempUnschedulable); const windowStats = isRecord(parsed.windowStats) ? parsed.windowStats : {}; + const diagnostics = isRecord(parsed.diagnostics) ? parsed.diagnostics : {}; + const timing = isRecord(diagnostics.timing) ? diagnostics.timing : {}; const logs = isRecord(parsed.logs) ? parsed.logs : {}; const accountSnapshot = recordArray(parsed.accountSnapshot); const lines: string[] = []; @@ -439,13 +441,23 @@ export function renderTraceReport( `events=${textValue(summary.eventCount)}`, `window=${window.beforeSeconds ?? "?"}s/${window.afterSeconds ?? "?"}s`, ].join(" ")); + lines.push(renderTable([ + ["TIME_REF", "FIRST_FAILOVER_MS", "SELECT_FAILED_MS", "FINAL_MS", "FAILOVER_TO_FINAL_MS", "CLIENT_DEADLINE", "REMAINING_MS"], + [ + textValue(timing.referenceSource), textValue(timing.firstFailoverElapsedMs), textValue(timing.firstSelectFailedElapsedMs), + textValue(timing.finalElapsedMs), textValue(timing.failoverToFinalMs), timing.clientDeadlineAvailable === true ? "available" : "unavailable", + textValue(timing.clientDeadlineRemainingMs), + ], + ])); + if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); if (failovers.length > 0) { lines.push(""); lines.push("FAILOVER"); lines.push(renderTable([ - ["AT", "ACCOUNT", "UPSTREAM", "SWITCH", "MAX"], + ["AT", "ELAPSED_MS", "ACCOUNT", "UPSTREAM", "SWITCH", "MAX"], ...failovers.map((item) => [ shortIso(item.at), + textValue(item.elapsedMs), formatAccountRef(item), textValue(item.upstreamStatus), textValue(item.switchCount), @@ -457,9 +469,10 @@ export function renderTraceReport( lines.push(""); lines.push("SELECT-FAILED"); lines.push(renderTable([ - ["AT", "ERROR", "EXCLUDED"], + ["AT", "ELAPSED_MS", "ERROR", "EXCLUDED"], ...selectFailures.map((item) => [ shortIso(item.at), + textValue(item.elapsedMs), shorten(stringValue(item.error) ?? "-", 56), textValue(item.excludedAccountCount), ]), @@ -469,9 +482,10 @@ export function renderTraceReport( lines.push(""); lines.push("FORWARD-FAILED"); lines.push(renderTable([ - ["AT", "ACCOUNT", "STATUS", "ERROR"], + ["AT", "ELAPSED_MS", "ACCOUNT", "STATUS", "ERROR"], ...forwardFailures.map((item) => [ shortIso(item.at), + textValue(item.elapsedMs), formatAccountRef(item), textValue(item.statusCode), shorten(stringValue(item.error) ?? "-", 72), diff --git a/scripts/src/platform-infra-sub2api-codex/runtime.ts b/scripts/src/platform-infra-sub2api-codex/runtime.ts index 103ca8b0..df540a44 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime.ts @@ -170,6 +170,8 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown nativeOps: { source: nativeOps.source, overview: nativeOps.overview, + responseHeaderTimeout: nativeOps.responseHeaderTimeout, + requestTiming: nativeOps.requestTiming, accountAvailability: { status: availability.status, totalAccounts: availabilityGroup.total_accounts, @@ -277,6 +279,8 @@ function renderRuntimeErrors(lines: string[], runtime: Record, const allGroups = runtimeRecord(runtime.allGroups); if (allGroups !== null) { const totals = runtimeRecord(allGroups.pageTotals) ?? {}; + const timeout = runtimeRecord(allGroups.responseHeaderTimeout) ?? {}; + const timing = runtimeRecord(allGroups.requestTiming) ?? {}; const groups = runtimeRecords(allGroups.groups); lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=all-groups since=${options.since}`); lines.push(renderTable([ @@ -287,13 +291,22 @@ function renderRuntimeErrors(lines: string[], runtime: Record, runtimeRate(totals.upstreamErrorRate), runtimeText(totals.groupsNeedingAttention), ], ])); + lines.push(renderTable([ + ["OPENAI_HEADER_TIMEOUT", "SOURCE", "GENERIC_TIMEOUT", "GENERIC_APPLIES_TO_OPENAI", "PAGE_MAX_TTFT_P99_MS", "OBSERVED_FLOOR_SECONDS"], + [ + timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source), + `${runtimeText(timeout.genericResponseHeaderTimeoutSeconds)}s`, runtimeText(timeout.genericAppliesToOpenAI), + runtimeText(timing.maxTtftP99Ms), runtimeText(timing.observedFloorSeconds), + ], + ])); + if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); lines.push("", "GROUPS"); lines.push(renderTable([ - ["NAME", "ID", "PLATFORM", "STATUS", "REQ", "CLIENT_ERR", "ERR_RATE", "UP_ERR", "UP_RATE", "ACCOUNTS", "AVAILABLE", "USE/CAP", "QUEUE", "ATTN"], + ["NAME", "ID", "PLATFORM", "STATUS", "REQ", "CLIENT_ERR", "ERR_RATE", "UP_ERR", "UP_RATE", "TTFT_P99_MS", "DUR_P99_MS", "ACCOUNTS", "AVAILABLE", "USE/CAP", "QUEUE", "ATTN"], ...groups.map((group) => [ runtimeShort(group.groupName, 28), runtimeText(group.groupId), runtimeText(group.platform), runtimeText(group.status), runtimeText(group.requestCount), runtimeText(group.errorCount), runtimeRate(group.errorRate), runtimeText(group.upstreamErrorCount), - runtimeRate(group.upstreamErrorRate), runtimeText(group.totalAccounts), runtimeText(group.availableCount), + runtimeRate(group.upstreamErrorRate), runtimeText(group.ttftP99Ms), runtimeText(group.durationP99Ms), runtimeText(group.totalAccounts), runtimeText(group.availableCount), `${runtimeText(group.currentInUse)}/${runtimeText(group.maxCapacity)}`, runtimeText(group.waitingInQueue), group.needsAttention === true ? "yes" : "no", ]), ])); @@ -308,6 +321,10 @@ function renderRuntimeErrors(lines: string[], runtime: Record, const overview = runtimeRecord(nativeOps.overview) ?? {}; const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; + const timeout = runtimeRecord(nativeOps.responseHeaderTimeout) ?? {}; + const timing = runtimeRecord(nativeOps.requestTiming) ?? {}; + const ttft = runtimeRecord(timing.ttft) ?? {}; + const duration = runtimeRecord(timing.duration) ?? {}; lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`); lines.push(renderTable([ ["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"], @@ -317,6 +334,23 @@ function renderRuntimeErrors(lines: string[], runtime: Record, runtimeText(overview.businessLimitedCount), runtimeText(overview.healthScore), ], ])); + lines.push(renderTable([ + ["OPENAI_HEADER_TIMEOUT", "SOURCE", "TTFT_P50_MS", "TTFT_P95_MS", "TTFT_P99_MS", "TTFT_MAX_MS", "DURATION_P99_MS", "DURATION_MAX_MS"], + [ + timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source), + runtimeText(ttft.p50_ms), runtimeText(ttft.p95_ms), runtimeText(ttft.p99_ms), runtimeText(ttft.max_ms), + runtimeText(duration.p99_ms), runtimeText(duration.max_ms), + ], + ])); + const candidates = runtimeRecords(timing.candidates); + if (candidates.length > 0) lines.push("", "TIMEOUT CANDIDATES", renderTable([ + ["SECONDS", "OBSERVED_FLOOR", "TTFT_P99_HEADROOM_MS", "TTFT_P99_BELOW", "ALL_DUR>=", "ERROR_DUR>=", "ASSESSMENT"], + ...candidates.map((candidate) => [ + runtimeText(candidate.seconds), runtimeText(candidate.observedFloor), runtimeText(candidate.ttftP99HeadroomMs), runtimeText(candidate.ttftP99Below), + runtimeText(candidate.totalDurationAtLeastCount), runtimeText(candidate.errorDurationAtLeastCount), runtimeText(candidate.assessment), + ]), + ])); + if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); lines.push(renderTable([ ["ACCOUNTS", "AVAILABLE", "RATE_LIMIT", "ERROR", "IN_USE", "CAPACITY", "QUEUE"], [ @@ -750,6 +784,69 @@ def runtime_log_lines(): output = proc.stdout + b"\\n" + proc.stderr return output.decode("utf-8", errors="replace").splitlines() +def app_exec(args): + if RUNTIME_MODE == "host-docker": + return docker(["exec", APP_POD, *args]) + return kubectl(["-n", NAMESPACE, "exec", APP_POD, "--", *args]) + +def app_text(args): + proc = app_exec(args) + return proc.stdout.decode("utf-8", errors="replace").strip() if proc.returncode == 0 else "" + +def config_scalar(path, key): + script = r'''awk -v wanted="$2" ' +BEGIN { in_gateway=0 } +/^gateway:[[:space:]]*($|#)/ { in_gateway=1; next } +in_gateway && /^[^[:space:]#]/ { exit } +in_gateway && $0 ~ "^[[:space:]]+" wanted ":[[:space:]]*" { + value=$0 + sub("^[[:space:]]+" wanted ":[[:space:]]*", "", value) + sub("[[:space:]]*#.*$", "", value) + gsub("^[[:space:]\\\"'\''']+|[[:space:]\\\"'\''']+$", "", value) + print value + exit +}' "$1"''' + return app_text(["sh", "-c", script, "sh", path, key]) + +def runtime_response_header_timeout(): + env_openai = app_text(["sh", "-c", "printenv GATEWAY_OPENAI_RESPONSE_HEADER_TIMEOUT 2>/dev/null || true"]) + env_generic = app_text(["sh", "-c", "printenv GATEWAY_RESPONSE_HEADER_TIMEOUT 2>/dev/null || true"]) + config_file = app_text(["sh", "-c", "printenv CONFIG_FILE 2>/dev/null || true"]) + paths = [value for value in (config_file, "/app/data/config.yaml", "/etc/sub2api/config.yaml", "/app/config.yaml") if value] + config_openai = "" + config_generic = "" + config_source = None + for path in dict.fromkeys(paths): + openai_value = config_scalar(path, "openai_response_header_timeout") + generic_value = config_scalar(path, "response_header_timeout") + if openai_value or generic_value: + config_openai = openai_value + config_generic = generic_value + config_source = path + break + def non_negative_int(value): + try: + parsed = int(str(value).strip()) + return parsed if parsed >= 0 else None + except (TypeError, ValueError): + return None + openai_configured = non_negative_int(config_openai) + openai_env = non_negative_int(env_openai) + generic_configured = non_negative_int(config_generic) + generic_env = non_negative_int(env_generic) + effective = openai_configured if openai_configured is not None else openai_env if openai_env is not None else 0 + source = "config-file:" + config_source if openai_configured is not None else "process-env" if openai_env is not None else "sub2api-default" + generic = generic_configured if generic_configured is not None else generic_env if generic_env is not None else 600 + return { + "effectiveSeconds": effective, + "disabled": effective == 0, + "source": source, + "genericResponseHeaderTimeoutSeconds": generic, + "genericAppliesToOpenAI": False, + "configFileChecked": config_source, + "valuesPrinted": False, + } + def compact_error(value): if not isinstance(value, str): return "" @@ -981,6 +1078,7 @@ def native_ops_snapshot(token, group_id, platform=None): "business_limited_count", "error_count_sla", "request_count_total", "request_count_sla", "error_rate", "upstream_error_rate", "upstream_error_count_excl_429_529", "upstream_429_count", "upstream_529_count", + "duration", "ttft", ) } @@ -1062,6 +1160,64 @@ def native_ops_snapshot(token, group_id, platform=None): "systemLogSink": {"status": sink_result.get("status"), "health": sink if isinstance(sink, dict) else None, "reason": sink_result.get("reason")}, } +def native_request_timing(token, group_id, platform, overview): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + base = f"group_id={group_id}&start_time={start_time}{platform_query}" + all_counts = {} + error_counts = {} + status = "available" + reason = None + duration = overview.get("duration") if isinstance(overview.get("duration"), dict) else {} + ttft = overview.get("ttft") if isinstance(overview.get("ttft"), dict) else {} + ttft_p99 = ttft.get("p99_ms") + observed_floor = max(60, ((ttft_p99 + 29999) // 15000) * 15) if isinstance(ttft_p99, int) else None + candidate_seconds = sorted(set([30, 45, 60] + ([observed_floor] if isinstance(observed_floor, int) else []))) + error_duration_response = curl_api("GET", f"/api/v1/admin/ops/requests?{base}&page=1&page_size=1&kind=error&min_duration_ms=0", bearer=token) + try: + error_duration_data = data_of(error_duration_response, "get error duration coverage") + error_duration_record_count = error_duration_data.get("total") if isinstance(error_duration_data, dict) else None + except Exception as exc: + error_duration_record_count = None + status = "unavailable" + reason = compact_error(str(exc)) + for seconds in candidate_seconds: + threshold = seconds * 1000 + for kind, target in ((None, all_counts), ("error", error_counts)): + kind_query = "&kind=error" if kind else "" + response = curl_api("GET", f"/api/v1/admin/ops/requests?{base}&page=1&page_size=1&min_duration_ms={threshold}{kind_query}", bearer=token) + try: + data = data_of(response, "get long request count") + value = data.get("total") if isinstance(data, dict) else None + target[threshold] = None if kind == "error" and error_duration_record_count == 0 else value + except Exception as exc: + status = "unavailable" + reason = compact_error(str(exc)) + target[threshold] = None + candidates = [] + for seconds in candidate_seconds: + headroom = seconds * 1000 - ttft_p99 if isinstance(ttft_p99, int) else None + candidates.append({ + "seconds": seconds, + "observedFloor": seconds == observed_floor, + "ttftP99HeadroomMs": headroom, + "ttftP99Below": headroom >= 0 if isinstance(headroom, int) else None, + "totalDurationAtLeastCount": all_counts.get(seconds * 1000), + "errorDurationAtLeastCount": error_counts.get(seconds * 1000), + "assessment": "observed-ttft-p99-plus-15s-rounded" if seconds == observed_floor else "ttft-p99-within-candidate" if isinstance(headroom, int) and headroom >= 0 else "ttft-p99-exceeds-candidate" if isinstance(headroom, int) else "insufficient-ttft-evidence", + }) + return { + "status": status, + "reason": reason, + "source": "sub2api-native-admin-ops-dashboard-and-requests", + "ttft": ttft, + "duration": duration, + "observedFloorSeconds": observed_floor, + "errorDurationRecordCount": error_duration_record_count, + "candidates": candidates, + "attribution": "TTFT covers successful streaming requests with first_token_ms. Total and error duration are end-to-end request durations, not response-header wait or guaranteed failover lead time. The observed floor is TTFT P99 plus 15 seconds rounded up to 15 seconds; it is not a deployment recommendation. A dash in ERROR_DUR>= means error duration coverage is unavailable.", + } + def all_group_error_overview(token, groups): page_size = 20 cursor = int(PAYLOAD.get("pageToken") or 0) @@ -1120,6 +1276,8 @@ def all_group_error_overview(token, groups): "upstreamErrorCount": upstream_error_count, "upstreamErrorRate": overview.get("upstream_error_rate"), "businessLimitedCount": business_limited, + "ttftP99Ms": ((overview.get("ttft") or {}).get("p99_ms") if isinstance(overview.get("ttft"), dict) else None), + "durationP99Ms": ((overview.get("duration") or {}).get("p99_ms") if isinstance(overview.get("duration"), dict) else None), "totalAccounts": availability_group.get("total_accounts"), "availableCount": availability_group.get("available_count"), "rateLimitCount": availability_group.get("rate_limit_count"), @@ -1138,6 +1296,9 @@ def all_group_error_overview(token, groups): next_token = str(page[-1]["id"]) if len(remaining) > page_size and page else None totals["errorRate"] = round(totals["errorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None totals["upstreamErrorRate"] = round(totals["upstreamErrorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None + ttft_p99_values = [item.get("ttftP99Ms") for item in summaries if isinstance(item.get("ttftP99Ms"), int)] + max_ttft_p99 = max(ttft_p99_values) if ttft_p99_values else None + observed_floor = max(60, ((max_ttft_p99 + 29999) // 15000) * 15) if isinstance(max_ttft_p99, int) else None return { "scope": "all-groups-overview", "window": {"since": PAYLOAD.get("since")}, @@ -1148,6 +1309,13 @@ def all_group_error_overview(token, groups): "pageToken": str(cursor) if cursor > 0 else None, "nextPageToken": next_token, "pageTotals": totals, + "responseHeaderTimeout": runtime_response_header_timeout(), + "requestTiming": { + "maxTtftP99Ms": max_ttft_p99, + "observedFloorSeconds": observed_floor, + "scope": "returned-groups-page", + "attribution": "The observed floor is the maximum returned-group TTFT P99 plus 15 seconds rounded up to 15 seconds. It is analytical only and must not be treated as a safe timeout.", + }, "groups": summaries, "disclosure": "Page totals cover only returned groups. Follow nextPageToken for remaining groups and use --group for account root causes and request indexes.", } @@ -1466,6 +1634,8 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): effectiveness = policy_effectiveness(lines, accounts, system_log_source.get("source"), group_id) effectiveness["sourceStatus"] = system_log_source overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) + native_ops["responseHeaderTimeout"] = runtime_response_header_timeout() + native_ops["requestTiming"] = native_request_timing(token, group_id, platform, overview_summary) request_total = overview_summary.get("request_count_total") error_total = overview_summary.get("error_count_total") if isinstance(request_total, int) and isinstance(error_total, int): @@ -1599,6 +1769,8 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): "upstreamErrorRate": overview.get("upstream_error_rate"), "businessLimitedCount": overview.get("business_limited_count"), }, + "responseHeaderTimeout": native_ops.get("responseHeaderTimeout"), + "requestTiming": native_ops.get("requestTiming"), "accountAvailability": { "status": (native_ops.get("accountAvailability") or {}).get("status"), "group": availability.get("group"), From da37a9642449e7edcf64a89ea898304de7f53346 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 07:07:48 +0200 Subject: [PATCH 3/4] feat: improve sub2api recent error analysis --- .../unidesk-sub2api/references/codex-pool.md | 7 ++ .../R2.2.6_Task_Report.md | 68 ++++++++++ docs/MDTODO/sub2api-upstream-reliability.md | 6 + .../remote-python-sync.ts | 47 ++++++- .../platform-infra-sub2api-codex/runtime.ts | 117 ++++++++++++++++++ 5 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.6_Task_Report.md diff --git a/.agents/skills/unidesk-sub2api/references/codex-pool.md b/.agents/skills/unidesk-sub2api/references/codex-pool.md index a9d7fdb5..4b2969ca 100644 --- a/.agents/skills/unidesk-sub2api/references/codex-pool.md +++ b/.agents/skills/unidesk-sub2api/references/codex-pool.md @@ -12,6 +12,11 @@ - `--platform ` 可筛选平台；省略分组参数时继续分析 YAML 当前 Codex pool； - 默认输出使用 Kubernetes 风格紧凑表格；机器读取必须显式使用 `--json`、`--full` 或 `--raw`； - dashboard 请求量和客户错误率按分组精确统计； + - 指定分组下钻同时返回原生 `openai-token-stats` 模型时延与 `request-errors` 客户错误画像： + - 模型时延只提供模型维度的平均 TTFT、平均总耗时和首 token 覆盖率； + - 客户错误画像提供模型、账号、状态码、同步/流式、阶段、入口和精确请求路径分布； + - 原生接口没有账号维度 TTFT 时禁止从错误账号分布推导账号 TTFT； + - 默认只给有界 request ID 索引，使用 `trace` 做精确下钻； - 账号被多个分组共享时，账号级 upstream-errors 和缺少 `group_id` 的 runtime 事件属于共享证据，禁止跨分组相加； - dashboard overview 提供请求分母和错误率； - dashboard overview 的 `ttft` 提供成功流式请求的首 token 分位数，`duration` 提供成功请求总耗时分位数；二者不能互相替代； @@ -109,6 +114,8 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 `trace --request-id ` 是只读 request 追溯报表，不触发 probe、不修改账号。命令优先使用原生 `/admin/ops/requests` 获取请求结果，并使用 `/admin/ops/system-logs` 获取 request-id 索引事件；target runtime 日志只补原生索引未保留的邻近事件。报表显式输出 target、runtime mode、主数据源、上下文数据源和读取状态。默认输出请求开始/最终状态、failover、`account_select_failed`、`forward_failed`、窗口内 `account_temp_unschedulable`、admin schedulable 写入计数和当前账号快照。`forward_failed` 与 HTTP 200 并存且没有 failover 时，结果标记为 `degraded`，不能解释为完整流式成功；`reason=failover-attempted-no-candidate` 表示 Sub2API 已进入自动切号，但排除当前失败账号后没有可用候选。需要机器处理时使用 `--raw`，需要原始匹配行时加 `--show-lines`。 +近期故障分析默认使用短窗口，并优先接受原生 Ops 证据。`trace` 在原生 requests 详情缺失时可在同一 `--since` 窗口内查询原生 request-errors；只有原生 system-log 索引缺失时才使用 runtime 日志补事件。历史日志已过保留期不构成运行面缺陷，也不应阻塞近期根因判断。 + `trace` 的时间线以原生索引事件时间戳为主。原生 request 记录只有完成时间、但最终日志带 `latency_ms` 时，命令用“完成时间减总耗时”重建参考起点，并披露首次 failover、选号失败和最终响应相对耗时。Sub2API Ops 没有客户 deadline 时，客户剩余时间必须显示不可用，不能用 `latency_ms` 推断。 对已支持的 k3s target，`sync --confirm`、`validate` 和 sentinel 操作可能超过单次 SSH/runtime 短连接窗口；远端 job/poll 边界见 [sentinel.md](sentinel.md)。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.6_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.6_Task_Report.md new file mode 100644 index 00000000..e9c8f159 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.6_Task_Report.md @@ -0,0 +1,68 @@ +# R2.2.6 任务报告 + +## 结果 + +- 完成纯分析 CLI 增强，未修改 Sub2API 配置、账号 runtime、YAML 或运行面。 +- `runtime errors --all-groups` 默认使用 Kubernetes 风格概要表，支持所有分组概览和 `--group ` 精确下钻。 +- 指定分组详情新增原生 `openai-token-stats` 模型时延表，以及原生 `request-errors` 的模型、账号、状态、同步/流式、阶段、入口和精确路径分布。 +- 客户错误画像由 CLI 自动读取原生分页并完成统计，只对 request ID 样本做有界渐进披露，不要求用户手工调整 limit。 +- `trace` 继续以原生 requests 和 system-logs 为主；requests 缺记录时只在同一 `--since` 窗口查询原生 request-errors，runtime 日志仅补原生索引缺失的事件。 + +## 最近 60 分钟全分组证据 + +- 三个分组合计约 468 个请求、11 个客户可见错误，客户错误率约 2.35%，上游错误率约 4.70%。 +- `unidesk-codex-pool`：约 61 至 67 个请求、7 个客户错误，窗口滚动错误率约 10.45% 至 11.48%。 +- `自用`：407 个请求、4 个客户错误，错误率 0.98%。 +- `grok`：窗口内无流量、无错误。 +- 查询时并发约 8/218、队列为 0；当前快照不能代表错误瞬间，但没有证据支持把全局并发不足作为近期主因。 + +## 分组根因 + +- `unidesk-codex-pool` 的 7 个客户错误全部是： + - 模型 `gpt-5.6-luna`； + - 账号 27； + - 同步 `/v1/responses`； + - 上游阶段； + - 5 个 502、2 个 503。 +- `自用` 的 4 个客户错误全部是： + - 模型 `gpt-5.6-sol`； + - 流式 `/responses`； + - 账号 15 占 3 个 504，账号 27 占 1 个 502。 +- 近期失败不是所有模型、账号和路径均匀恶化，而是两个窄错误簇。 + +## request trace 证据 + +- `c76b05df-ad84-4055-bb5a-d363a77e94a1`：约 197.056 秒后账号 27 返回 503，首次 failover、下一次选号失败和最终 502 发生在同一毫秒，选号错误为 `context canceled`。 +- `2c1211b5-1f9f-441a-9ef8-1f80795097cd`：约 50.863 秒后账号 27 返回 502，failover 与 `context canceled` 同时发生。 +- `d66d7ea7-4e78-475c-8f93-b315435b79f7`：约 1772.971 秒后账号 15 返回上游 504，随后立即 `context canceled` 并包装成最终 502。 +- `4ee54daa-f41c-4f50-bd72-11080c38e299`：约 730.835 秒后账号 27 返回 502，约 1 毫秒后 `context canceled`。 +- 四个近期代表请求都已进入 failover；临时不可调度规则也有真实命中。直接传播点是 failover 开始时原请求上下文已经取消，不是缺少规则或没有触发切号。 + +## v0.1.153 官方源码判断 + +- OpenAI Responses handler 的转发和下一次账号选择都复用 `c.Request.Context()`。 +- 收到 `UpstreamFailoverError` 后 handler 记录失败账号并直接继续选号；若客户请求上下文已经取消，下一次选择会立即返回 `context canceled`。 +- `gateway.openai_response_header_timeout` 是 v0.1.153 已有配置，作用范围是等待 OpenAI 上游 HTTP 响应头，不影响响应头之后的流式传输。 +- v0.1.153 发布说明没有修复上述 failover 上下文生命周期问题，因此没有证据支持回退版本。 + +## 候选方案 + +1. 配置级有界缓解，不涉及 Sub2API 源码变更： + - 设置 OpenAI 专用 response-header timeout，可让真正卡在响应头前的 730 秒或 1773 秒请求更早转成 failover，从而保留切号窗口。 + - 它不能解决 50 至 197 秒内已经返回的 502/503，也不能解决流式内容已经提交后的失败。 + - 近期 TTFT P99 在分组 2 约 231 至 245 秒、分组 3 约 101 秒；TTFT 不是响应头时延，因此 255/270 秒只能作为排除过低值的观测下限，不能直接视为安全配置。 +2. 精确源码方案，涉及 Sub2API 源码变更： + - 增加响应头实际等待时长与首个语义流事件时长，区分响应头前等待和响应头后无事件。 + - 在下游尚未提交且客户 deadline 仍有预算时，为单次上游尝试使用更短的 attempt context；超时后在原客户 context 内切号。 + - 明确禁止在客户已经断开后继续后台重试，避免无效上游消耗。 +3. 窄路由治理： + - 分组 2 优先治理账号 27 的 `gpt-5.6-luna` 能力和错误分类；分组 3 优先治理账号 15/27 的 `gpt-5.6-sol` 长流式等待。 + - 不建议用降低全池并发、账号 pinning、扩大临时不可调度时长或版本回退替代根因治理。 + +## 验证 + +- `bun --check` 通过：`runtime.ts`、`remote-python-sync.ts`、`output.ts`。 +- `bun test scripts/src/output.test.ts`：2 通过、0 失败。 +- `git diff --check` 通过。 +- 原入口读取验证通过：全分组概要、分组 2/3 详情、四个近期 request trace。 +- 超过 3000 行的既有 `remote-python-sync.ts` 拆分已登记为 R2.2.7，本任务未混入大范围机械重构。 diff --git a/docs/MDTODO/sub2api-upstream-reliability.md b/docs/MDTODO/sub2api-upstream-reliability.md index ef547536..3a3c36ba 100644 --- a/docs/MDTODO/sub2api-upstream-reliability.md +++ b/docs/MDTODO/sub2api-upstream-reliability.md @@ -35,6 +35,12 @@ #### R2.2.5 [completed] 增强 Sub2API CLI 的纯分析能力：只读披露实际 OpenAI response-header timeout、TTFT 分位数、长等待请求和 failover 剩余时间窗，并使用原生 admin/ops 数据分析候选配置，不修改 Sub2API 配置、源码或运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.5_Task_Report.md)。 +#### R2.2.6 [completed] + +继续纯分析 Sub2API timeout 与 failover：按所有分组、模型、账号和请求类型定位超长 TTFT 与客户错误，区分响应头前等待、流式提交后失败和无候选，评估观测下限是否可能缓解且不做配置实验或运行面变更，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.6_Task_Report.md)。 +#### R2.2.7 + +按仓库 P0 拆分超过 3000 行的 `remote-python-sync.ts`：把大段内嵌 Python 按 trace、sync/validate 与 sentinel 职责迁入原生模块，单文件收敛到 2000 行以下，并保持受控 CLI 行为与只读诊断契约不变，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.7_Task_Report.md)。 ### R2.3 调查并收敛 Sub2API 原生 system-log 索引未保留策略事件的能力缺口，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.3_Task_Report.md)。 diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts index 47c2374d..a92d20eb 100644 --- a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts @@ -3006,6 +3006,16 @@ def trace_untried_schedulable_accounts(failovers, final_event, account_snapshot) }) return result +def ops_start_time_for_since(since): + if not isinstance(since, str): + return None + match = re.fullmatch(r"\s*(\d+)\s*([smhd])\s*", since.lower()) + if match is None: + return None + value = int(match.group(1)) + multiplier = {"s": 1, "m": 60, "h": 3600, "d": 86400}[match.group(2)] + return (datetime.now(timezone.utc) - timedelta(seconds=value * multiplier)).strftime("%Y-%m-%dT%H:%M:%SZ") + def run_trace(): payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {} request_id = payload.get("requestId") @@ -3032,6 +3042,35 @@ def run_trace(): "returned": len(request_items), "total": request_data.get("total", len(request_items)) if isinstance(request_data, dict) else len(request_items), } + if native_request is None: + ops_start_time = ops_start_time_for_since(since) + error_query = f"/api/v1/admin/ops/request-errors?q={request_query}&view=all&page=1&page_size=20" + if isinstance(ops_start_time, str): + error_query += "&start_time=" + quote(ops_start_time, safe="") + error_data = ensure_success( + curl_api("GET", error_query, bearer=token), + "get ops request error fallback", + ) + error_items = extract_items(error_data) + native_request = next((item for item in error_items if isinstance(item, dict) and item.get("request_id") == request_id), None) + native_request_status.update({ + "fallback": "sub2api-native-admin-ops-request-errors", + "fallbackStartTime": ops_start_time, + "fallbackReturned": len(error_items), + "fallbackTotal": error_data.get("total", len(error_items)) if isinstance(error_data, dict) else len(error_items), + }) + if isinstance(native_request, dict): + native_request = dict(native_request) + native_request["kind"] = "error" + error_id = native_request.get("id") + if isinstance(error_id, int): + try: + detail = ensure_success(curl_api("GET", f"/api/v1/admin/ops/request-errors/{error_id}", bearer=token), "get ops request error detail") + if isinstance(detail, dict): + native_request.update(detail) + except Exception: + pass + native_request_status.update({"matched": True, "source": "sub2api-native-admin-ops-request-errors"}) except Exception as exc: native_request_status = {"status": "unavailable", "reason": text(str(exc), 500)} @@ -3119,7 +3158,7 @@ def run_trace(): "type": "request-start", "at": native_request.get("created_at"), "requestId": native_request.get("request_id"), - "path": None, + "path": native_request.get("request_path") or native_request.get("inbound_endpoint"), "model": native_request.get("model"), "accountId": native_request.get("account_id"), "accountName": account_names_by_id.get(native_request.get("account_id")), @@ -3137,7 +3176,7 @@ def run_trace(): "type": "final", "at": native_request.get("created_at"), "requestId": native_request.get("request_id"), - "path": None, + "path": native_request.get("request_path") or native_request.get("inbound_endpoint"), "model": native_request.get("model"), "accountId": native_request.get("account_id"), "accountName": account_names_by_id.get(native_request.get("account_id")), @@ -3182,7 +3221,7 @@ def run_trace(): final_elapsed = round((final_epoch - reference_epoch) * 1000) if final_epoch is not None and reference_epoch is not None else None failover_to_final = final_elapsed - first_failover_elapsed if isinstance(final_elapsed, int) and isinstance(first_failover_elapsed, int) else None reason = trace_reason(events, final_event) - if not matched: + if not matched and not isinstance(native_request, dict): outcome = "not-found" elif isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400: outcome = "degraded" if forward_failures else ("succeeded-with-failover" if failovers else "succeeded") @@ -3191,7 +3230,7 @@ def run_trace(): else: outcome = "incomplete" return { - "ok": proc.returncode == 0 and len(matched) > 0, + "ok": proc.returncode == 0 and (len(matched) > 0 or isinstance(native_request, dict)), "mode": "trace", "targetId": TARGET_ID, "runtimeMode": RUNTIME_MODE, diff --git a/scripts/src/platform-infra-sub2api-codex/runtime.ts b/scripts/src/platform-infra-sub2api-codex/runtime.ts index df540a44..077d1661 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime.ts @@ -172,6 +172,7 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown overview: nativeOps.overview, responseHeaderTimeout: nativeOps.responseHeaderTimeout, requestTiming: nativeOps.requestTiming, + customerErrors: nativeOps.customerErrors, accountAvailability: { status: availability.status, totalAccounts: availabilityGroup.total_accounts, @@ -325,6 +326,7 @@ function renderRuntimeErrors(lines: string[], runtime: Record, const timing = runtimeRecord(nativeOps.requestTiming) ?? {}; const ttft = runtimeRecord(timing.ttft) ?? {}; const duration = runtimeRecord(timing.duration) ?? {}; + const customerErrors = runtimeRecord(nativeOps.customerErrors) ?? {}; lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`); lines.push(renderTable([ ["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"], @@ -351,6 +353,32 @@ function renderRuntimeErrors(lines: string[], runtime: Record, ]), ])); if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); + const modelTiming = runtimeRecords(timing.modelTiming); + if (modelTiming.length > 0) lines.push("", "MODEL TIMING", renderTable([ + ["MODEL", "REQUESTS", "FIRST_TOKEN_ROWS", "COVERAGE", "AVG_TTFT_MS", "AVG_DURATION_MS", "AVG_TOKENS_S"], + ...modelTiming.map((item) => [runtimeShort(item.model, 32), runtimeText(item.requestCount), runtimeText(item.requestsWithFirstToken), runtimeRate(item.firstTokenCoverage), runtimeText(item.avgFirstTokenMs), runtimeText(item.avgDurationMs), runtimeText(item.avgTokensPerSec)]), + ])); + const errorModels = runtimeRecords(customerErrors.modelBuckets); + const errorAccounts = runtimeRecords(customerErrors.accountBuckets); + const errorDimensions = [ + ...runtimeRecords(customerErrors.statusBuckets).map((item) => ["status", runtimeText(item.value), runtimeText(item.count)]), + ...runtimeRecords(customerErrors.streamBuckets).map((item) => ["mode", runtimeText(item.value), runtimeText(item.count)]), + ...runtimeRecords(customerErrors.phaseBuckets).map((item) => ["phase", runtimeText(item.value), runtimeText(item.count)]), + ...runtimeRecords(customerErrors.endpointBuckets).map((item) => ["endpoint", runtimeText(item.value), runtimeText(item.count)]), + ...runtimeRecords(customerErrors.pathBuckets).map((item) => ["path", runtimeText(item.value), runtimeText(item.count)]), + ]; + if (errorModels.length > 0) lines.push("", `CUSTOMER ERRORS source=${runtimeText(customerErrors.source)} total=${runtimeText(customerErrors.total)}`, renderTable([ + ["MODEL", "COUNT"], ...errorModels.map((item) => [runtimeShort(item.value, 40), runtimeText(item.count)]), + ])); + if (errorAccounts.length > 0) lines.push(renderTable([ + ["ACCOUNT_ID", "COUNT"], ...errorAccounts.map((item) => [runtimeText(item.value), runtimeText(item.count)]), + ])); + if (errorDimensions.length > 0) lines.push(renderTable([["DIMENSION", "VALUE", "COUNT"], ...errorDimensions])); + const errorSamples = runtimeRecords(customerErrors.samples); + if (errorSamples.length > 0) lines.push("CUSTOMER ERROR INDEX", renderTable([ + ["REQUEST_ID", "MODEL", "STATUS", "ACCOUNT", "MODE", "PATH", "MESSAGE"], + ...errorSamples.map((item) => [runtimeText(item.requestId), runtimeShort(item.model, 26), runtimeText(item.statusCode), runtimeText(item.accountId), runtimeText(item.mode), runtimeShort(item.path, 34), runtimeShort(item.message, 48)]), + ])); lines.push(renderTable([ ["ACCOUNTS", "AVAILABLE", "RATE_LIMIT", "ERROR", "IN_USE", "CAPACITY", "QUEUE"], [ @@ -1206,6 +1234,32 @@ def native_request_timing(token, group_id, platform, overview): "errorDurationAtLeastCount": error_counts.get(seconds * 1000), "assessment": "observed-ttft-p99-plus-15s-rounded" if seconds == observed_floor else "ttft-p99-within-candidate" if isinstance(headroom, int) and headroom >= 0 else "ttft-p99-exceeds-candidate" if isinstance(headroom, int) else "insufficient-ttft-evidence", }) + since = str(PAYLOAD.get("since") or "") + token_range = {"30m": "30m", "60m": "1h", "1h": "1h", "24h": "1d", "1d": "1d"}.get(since) + model_timing = [] + model_timing_status = "unsupported-window" + if token_range: + response = curl_api("GET", f"/api/v1/admin/ops/dashboard/openai-token-stats?group_id={group_id}&time_range={token_range}&top_n=100{platform_query}", bearer=token) + try: + data = data_of(response, "get OpenAI model token stats") + rows = items_of(data) + for row in rows: + if not isinstance(row, dict): + continue + request_count = row.get("request_count") + first_count = row.get("requests_with_first_token") + model_timing.append({ + "model": row.get("model"), "requestCount": request_count, + "requestsWithFirstToken": first_count, + "firstTokenCoverage": round(first_count / request_count, 6) if isinstance(first_count, int) and isinstance(request_count, int) and request_count > 0 else None, + "avgFirstTokenMs": row.get("avg_first_token_ms"), "avgDurationMs": row.get("avg_duration_ms"), + "avgTokensPerSec": row.get("avg_tokens_per_sec"), + }) + model_timing.sort(key=lambda item: (-(item.get("avgFirstTokenMs") or -1), str(item.get("model") or ""))) + model_timing_status = "available" + except Exception as exc: + model_timing_status = "unavailable" + reason = compact_error(str(exc)) return { "status": status, "reason": reason, @@ -1214,10 +1268,71 @@ def native_request_timing(token, group_id, platform, overview): "duration": duration, "observedFloorSeconds": observed_floor, "errorDurationRecordCount": error_duration_record_count, + "modelTimingStatus": model_timing_status, + "modelTimingWindow": token_range, + "modelTiming": model_timing[:20 if PAYLOAD.get("full") is True else 8], "candidates": candidates, "attribution": "TTFT covers successful streaming requests with first_token_ms. Total and error duration are end-to-end request durations, not response-header wait or guaranteed failover lead time. The observed floor is TTFT P99 plus 15 seconds rounded up to 15 seconds; it is not a deployment recommendation. A dash in ERROR_DUR>= means error duration coverage is unavailable.", } +def native_customer_error_profile(token, group_id, platform): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + page = 1 + page_size = 500 + rows = [] + total = None + try: + while True: + response = curl_api("GET", f"/api/v1/admin/ops/request-errors?group_id={group_id}&start_time={start_time}&view=all&page={page}&page_size={page_size}{platform_query}", bearer=token) + data = data_of(response, "get customer-visible request errors") + page_rows = items_of(data) + rows.extend(page_rows) + if total is None and isinstance(data, dict) and isinstance(data.get("total"), int): + total = data.get("total") + if not page_rows or len(page_rows) < page_size or (isinstance(total, int) and len(rows) >= total): + break + page += 1 + except Exception as exc: + return {"status": "unavailable", "source": "sub2api-native-admin-ops-request-errors", "reason": compact_error(str(exc))} + if total is None: + total = len(rows) + def buckets(key_fn): + counts = {} + for row in rows: + value = key_fn(row) if isinstance(row, dict) else None + label = str(value) if value not in (None, "") else "unknown" + counts[label] = counts.get(label, 0) + 1 + return [{"value": key, "count": count} for key, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20 if PAYLOAD.get("full") is True else 8]] + samples = [] + seen = set() + for row in rows: + if not isinstance(row, dict) or not row.get("request_id"): + continue + key = (row.get("requested_model") or row.get("model"), row.get("status_code"), row.get("account_id"), row.get("stream"), row.get("request_path") or row.get("inbound_endpoint")) + if key in seen: + continue + seen.add(key) + samples.append({ + "requestId": row.get("request_id"), "model": key[0], "statusCode": key[1], "accountId": key[2], + "mode": "stream" if key[3] is True else "sync" if key[3] is False else "unknown", "path": key[4], + "message": compact_error(row.get("message") or row.get("error_message")), + }) + if len(samples) >= (30 if PAYLOAD.get("full") is True else 15): + break + return { + "status": "available", "source": "sub2api-native-admin-ops-request-errors", "total": total, + "returned": len(rows), "pagesRead": page, "recordsTruncated": False, + "modelBuckets": buckets(lambda row: row.get("requested_model") or row.get("model")), + "accountBuckets": buckets(lambda row: row.get("account_id")), + "statusBuckets": buckets(lambda row: row.get("status_code")), + "streamBuckets": buckets(lambda row: "stream" if row.get("stream") is True else "sync" if row.get("stream") is False else "unknown"), + "phaseBuckets": buckets(lambda row: row.get("phase")), + "endpointBuckets": buckets(lambda row: row.get("inbound_endpoint") or row.get("request_path")), + "pathBuckets": buckets(lambda row: row.get("request_path") or row.get("inbound_endpoint")), + "samples": samples, + } + def all_group_error_overview(token, groups): page_size = 20 cursor = int(PAYLOAD.get("pageToken") or 0) @@ -1636,6 +1751,7 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) native_ops["responseHeaderTimeout"] = runtime_response_header_timeout() native_ops["requestTiming"] = native_request_timing(token, group_id, platform, overview_summary) + native_ops["customerErrors"] = native_customer_error_profile(token, group_id, platform) request_total = overview_summary.get("request_count_total") error_total = overview_summary.get("error_count_total") if isinstance(request_total, int) and isinstance(error_total, int): @@ -1771,6 +1887,7 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): }, "responseHeaderTimeout": native_ops.get("responseHeaderTimeout"), "requestTiming": native_ops.get("requestTiming"), + "customerErrors": native_ops.get("customerErrors"), "accountAvailability": { "status": (native_ops.get("accountAvailability") or {}).get("status"), "group": availability.get("group"), From 4b55fdb711249011e3db7968caf6738808da9365 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:25:13 +0200 Subject: [PATCH 4/4] feat: preserve parallel sub2api diagnostics work --- .../unidesk-sub2api/references/codex-pool.md | 18 + .../references/troubleshooting-accounts.md | 6 + .agents/skills/unidesk-webdev/SKILL.md | 7 + config/platform-infra/sub2api-codex-pool.yaml | 12 +- .../R2.2.10_Task_Report.md | 53 + .../R2.2.11_Task_Report.md | 68 + .../R2.2.12_Task_Report.md | 66 + .../R2.2.13_Task_Report.md | 61 + .../R2.2.15_Task_Report.md | 14 + .../R2.2.16_Task_Report.md | 23 + .../R2.2.18_Task_Report.md | 26 + .../R2.2.20_Task_Report.md | 42 + .../R2.2.22_Task_Report.md | 59 + .../R2.2.24_Task_Report.md | 47 + .../R2.2.8_Task_Report.md | 55 + .../R2.2.9_Task_Report.md | 48 + docs/MDTODO/sub2api-upstream-reliability.md | 54 + .../runtime-options.ts | 120 ++ .../platform-infra-sub2api-codex/runtime.ts | 1180 ++++++++++++++--- 19 files changed, 1789 insertions(+), 170 deletions(-) create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.10_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.11_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.12_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.13_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.15_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.16_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.18_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.20_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.22_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.24_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.8_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.9_Task_Report.md create mode 100644 scripts/src/platform-infra-sub2api-codex/runtime-options.ts diff --git a/.agents/skills/unidesk-sub2api/references/codex-pool.md b/.agents/skills/unidesk-sub2api/references/codex-pool.md index 4b2969ca..2514dbc6 100644 --- a/.agents/skills/unidesk-sub2api/references/codex-pool.md +++ b/.agents/skills/unidesk-sub2api/references/codex-pool.md @@ -6,6 +6,11 @@ - PK01 路径不创建 k8s Secret、不部署 sentinel 资源，也不触发 `sub2api apply`、Docker compose、Caddy reload 或容器重启； - `runtime list|get` 查询 pool group 内全部账号或指定账号的实际状态；`get` 同时披露脱敏 `modelMapping` 和完整临时不可调度规则； - `runtime models --account` 通过 Sub2API 原生账号模型接口同时读取配置模型和上游实时模型，不把配置映射误当作真实上游能力； + - `runtime infrastructure --account` 只读关联账号 proxy 绑定与基础设施状态： + - 同时披露账号 proxy 记录和容器代理环境，不能因容器没有 `HTTP_PROXY` 就把账号误判为直连； + - 采集容器网络模式、DNS、默认路由、接口计数器、proxy socket、监听进程、服务状态和近期错误日志； + - 不发送上游账号测试或网络探针，不修改账号、proxy、调度或 YAML； + - socket、接口计数器和服务状态是查询时快照，不能单独证明历史故障窗口内网络正常； - `runtime errors` 优先使用 Sub2API 原生 admin/ops 能力： - `--all-groups` 按稳定 group ID 游标分页返回所有分组概要； - `--group ` 返回指定分组的账号根因、策略效果和 request 索引； @@ -17,6 +22,17 @@ - 客户错误画像提供模型、账号、状态码、同步/流式、阶段、入口和精确请求路径分布； - 原生接口没有账号维度 TTFT 时禁止从错误账号分布推导账号 TTFT； - 默认只给有界 request ID 索引，使用 `trace` 做精确下钻； + - 指定分组和 `--account` 下钻提供上游账号质量评分： + - 成功请求分母和账号级 TTFT 来自原生 admin usage； + - 客户错误来自原生 request-errors； + - 切号、同账号重试、临时不可调度和 `forward_failed` 来自原生 system-log 索引； + - 默认权重为可靠性 60、TTFT 25、当前可用性 15，并同时输出评分组成、等级、置信度和扣分原因； + - TTFT 少于 5 个样本时不进入分母，不能把缺失时延当成零时延； + - 至少 10 次账号尝试且至少 5 个 TTFT 样本时，才允许把高分映射为 A-D 可比较等级；样本不足的偶然高分标记为 `insufficient`； + - 至少 10 次账号尝试且评分低于 60 时，即使 TTFT 缺失也保留 E 级，避免明确的高失败率被“证据不足”掩盖； + - 请求证据不足时使用低置信度，不能据此自动调整 priority、capacity 或 schedulable； + - 账号名称是人类可读运维标识，应与账号 ID 一起完整输出；API Key、token 和 credentials 仍必须脱敏； + - 只有带 `group_id` 的 failover 和 `forward_failed` request ID 才进入分组账号失败率；账号级临时不可调度和同账号重试可以作为共享运维信号展示，但不加入分组失败分子； - 账号被多个分组共享时，账号级 upstream-errors 和缺少 `group_id` 的 runtime 事件属于共享证据，禁止跨分组相加； - dashboard overview 提供请求分母和错误率； - dashboard overview 的 `ttft` 提供成功流式请求的首 token 分位数，`duration` 提供成功请求总耗时分位数；二者不能互相替代； @@ -42,6 +58,7 @@ - 批量操作先校验全部 selector 存在且账号类型可修改，再顺序调用 Sub2API admin API： - 选择错误不会产生部分写入； - 上游 admin API 在批次中途失败时仍需逐账号 `get` 对账； + - runtime 模板必须按可复用错误语义设计，不得包含账号名称、账号 ID、provider URL 或分组特例；非 auth 错误使用精确状态码和稳定短语，auth 与账号状态错误保留专门处理路径； - 写操作不带 `--confirm` 时只返回 dry-run，带 `--confirm` 才通过 Sub2API admin API 写入； - 批量默认输出账号 ID、名称、change、mode、mutation 和汇总计数；runtime 命令不输出 API key 或完整 credentials，使用 `get`、`--full` 或 `--raw` 下钻； - 文档中的 runtime action 或参数被当前分支 CLI 拒绝时，按 CLI 集成漂移处理： @@ -58,6 +75,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool validate --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool runtime list --target PK01 bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --target PK01 --account bun scripts/cli.ts platform-infra sub2api codex-pool runtime models --target PK01 --account +bun scripts/cli.ts platform-infra sub2api codex-pool runtime infrastructure --target PK01 --account --since 30m bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h --all-groups bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h --group diff --git a/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md b/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md index 461a3359..5591c9c1 100644 --- a/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md +++ b/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md @@ -20,6 +20,12 @@ - 没有账号真实支持目标模型时，保持池不声明该模型并快速失败，不映射到名称相近的其他模型。 - 上游模型同步失败只能标记证据不足，不能据此补映射。 5. 调优按所有权下发： + - 非 auth 上游错误优先抽象为通用临时不可调度模板： + - 规则同时使用真实状态码和已证实的稳定响应短语； + - 先从当前 Sub2API 源码确认规则命中会在响应提交前触发当前请求 failover； + - 禁止在模板、CLI 或源码中硬编码账号名称、账号 ID、provider URL 或分组特例； + - 禁止用通用 400、通用 `invalid request` 或宽泛关键词覆盖真实客户参数错误； + - auth、token revoked、余额不足、分组删除和凭据失效使用专门的认证与账号状态路径，不混入通用非 auth 模板； - 通用模板先修改 owning YAML，并保持冷却保守；默认从一分钟开始，没有新证据和明确授权不得超过三分钟。 - YAML-managed 账号走 `sync`；runtime-manual 账号走显式 `runtime apply|delete`。 - 多账号使用 `--accounts` 精准集合；例外账号从 selector 中显式排除，不使用隐式全量操作。 diff --git a/.agents/skills/unidesk-webdev/SKILL.md b/.agents/skills/unidesk-webdev/SKILL.md index 856872fc..86f7c4d2 100644 --- a/.agents/skills/unidesk-webdev/SKILL.md +++ b/.agents/skills/unidesk-webdev/SKILL.md @@ -16,6 +16,13 @@ description: UniDesk Web 开发与浏览器验证技能。用户处理 UniDesk/H - 调试 HWLAB Cloud Web/Workbench 业务或功能 bug 时，使用 YAML 声明的 `--origin internal`；验收 public exposure、DNS、FRP、Caddy 或公网用户入口时，显式使用 `--origin public`。 - `--url` 只是 custom/local 一次性逃生口，与 `--origin` 互斥；禁止通过手写 URL 或 IP 选择内网/公网运行面。本地 fake-server、localhost、dist 静态服务和 custom URL 只能作为开发 preflight 证据。 - 禁止在本地或 master server 直接跑 `vue-tsc` / 前端全量 typecheck 作为默认验证；本地只做语法级检查和真实入口复测，完整类型检查交给 CI、PipelineRun 或明确指定的受控构建运行面。 +- HWLAB Cloud Web 的 Vitest 防误用规则： + - 以 HWLAB `docs/reference/web-test-resource-guard.md` 为权威； + - 配置必须默认关闭 watch、固定单 worker、限制 worker V8 堆； + - 测试、hook 和 teardown 必须设置有界 timeout； + - 正式 `test` 入口必须使用 `vitest run`、限制父进程 V8 堆并设置整体执行时限； + - Master server 只允许解析配置和检查 diff，不运行测试； + - Vitest 配置不能替代系统 cgroup `MemoryMax`。 - 桌面版 Web 截图、视觉复测和 web-probe 采样默认使用 `1920x1080` 视口；只有用户、issue 或 SPEC 明确指定其他尺寸时才覆盖，并在证据中写明覆盖原因。 - Web probe 命令和历史判定口径见 [references/web-probe.md](references/web-probe.md)；Playwright/fake-server 复现见 [references/playwright.md](references/playwright.md)。 - 前端改动遵循仓库既有设计系统、[references/design.md](references/design.md) 和 `$frontend-design` 全局 UI 规则；不要做营销式落地页替代真实工具页面。 diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 9b9702ae..d1c1f343 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -17,9 +17,13 @@ runtime: enabled: true rules: - statusCode: 429 - keywords: [concurrency limit exceeded] + keywords: [concurrency limit exceeded, upstream rate limit exceeded] durationMinutes: 1 - description: 明确命中上游并发限制时短时冷却当前账号。 + description: 明确命中上游并发或速率限制时短时冷却当前账号。 + - statusCode: 400 + keywords: [input must be a list] + durationMinutes: 1 + description: 明确命中 Responses input 列表兼容错误时短时冷却并切换账号。 - statusCode: 500 keywords: [failed to validate api key, do request failed] durationMinutes: 1 @@ -32,6 +36,10 @@ runtime: keywords: [upstream service temporarily unavailable, upstream request failed, service temporarily unavailable, temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 description: 明确命中 503 上游暂时不可用、过载或并发限制时短时冷却当前账号。 + - statusCode: 504 + keywords: [concurrency limit exceeded, upstream service temporarily unavailable, input must be a list] + durationMinutes: 1 + description: 明确命中 504 上游并发限制、暂时不可用或 Responses input 列表兼容错误时短时冷却当前账号。 - statusCode: 524 keywords: [upstream request failed, service temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.10_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.10_Task_Report.md new file mode 100644 index 00000000..5e0a0715 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.10_Task_Report.md @@ -0,0 +1,53 @@ +# R2.2.10 任务报告 + +## 结论 + +- 最近 30 分钟复取时，账号 `lyon9801`（ID 15）共 37 次尝试、37 次成功、0 次失败，TTFT P95 为 9040 ms；“自用”分组唯一客户错误来自账号 27 的流式 504。因此 `lyon9801` 的 503 是更早的孤立事件，不是当前持续不稳定。 +- `lyon9801` 不是 Sub2API 直连账号。它绑定 proxy 3：`platform-infra-sub2api-pk01-local-egress-proxy`，HTTP endpoint 为 `127.0.0.1:10809`。实际路径是 PK01 `sub2api-app` → PK01 本地 Hysteria 客户端 → NC01 Hysteria 服务端 → OpenAI。 +- 代表请求 `da415526-b124-4e23-a4a7-ea48154c851e` 在请求开始约 1556485 ms 后收到上游 HTTP 503。Sub2API 随即进入 failover，但 1 ms 后选号因 `context canceled` 失败，最终返回 502。客户影响的主要放大器是错误发现过晚，切号时已无可执行上下文，而不是 failover 未实现。 +- 另一请求 `4bba7f9b-15e3-4891-b063-bd65fb6e3704` 在 HTTP 200 后出现上游 `response.failed`，现由 trace 正确标记为 `degraded / forward-failed-http-success`。这是响应已提交后的流内失败，不能安全切号，也不等同于 proxy 连接失败。 +- 当前证据更支持 OpenAI/Codex 上游返回 HTTP 503 或应用层流失败，不支持 PK01 DNS、proxy connect、TCP reset、容器重启、OOM 或 NC01 Hysteria 重启为主因。历史瞬时丢包仍不能凭当前快照绝对排除。 + +## 基础设施证据 + +- PK01 `sub2api-app`：running，restart 0，OOM false，host network。 +- 账号 proxy 3：active，`127.0.0.1:10809`；当前有 4 条 established 本地 TCP socket。 +- PK01 proxy owner：`hysteria`，`host-proxy-hysteria.service` active/running，restart 0；最近两小时 journal 未命中 reconnect、disconnect、timeout 或 error。 +- PK01 DNS 为 `127.0.0.53`，默认路由经 `eth0 / 10.0.8.1`；可见接口 RX/TX error/drop 均为 0。 +- NC01 服务端容器：`vpn-hysteria2`，镜像 `tobyxdd/hysteria:v2.9.3`，running，restart 0，OOM false，host network；运行文件实际位于 `/root/vpn-server`。故障时刻附近没有对应 tunnel disconnect/reconnect 日志。 +- 账号 15 的近期 Sub2API 网络错误索引没有 DNS failure、proxy connect failure、connection reset、unexpected EOF、broken pipe、connection refused、network unreachable 或 transport timeout。 + +## Sub2API v0.1.153 源码归因 + +- 账号存在 `ProxyID` 时，OpenAI forward 把账号 proxy URL 传给 HTTP upstream；HTTP proxy 通过 `http.Transport.Proxy` 生效。因此容器没有 `HTTP_PROXY` 环境变量不能证明账号直连。 +- HTTP 503 属于已收到 HTTP response 的路径；所有 5xx 会构造 `UpstreamFailoverError` 并触发账号切换。 +- proxy、DNS、TCP 或 TLS 在 HTTP round trip 前失败时走 transport-error 路径，Ops status 为 0，最终 failover error 为 502。账号 15 的代表证据是 upstream status 503，不符合该传输故障路径。 +- OpenAI response-header timeout 在 v0.1.153 默认是 0；PK01 运行实例也显示 disabled。它允许请求长时间等待，但现有 Ops 只有端到端耗时，不能把 1556485 ms 全部断言为等待响应头。 +- 客户 context 已取消时，源码明确不再 failover；这解释了 503 后 1 ms 即 `account_select_failed: context canceled`。 + +## CLI 增强 + +新增只读入口： + +```bash +bun scripts/cli.ts platform-infra sub2api codex-pool runtime infrastructure \ + --target PK01 --group 3 --account lyon9801 --since 30m +``` + +默认 Kubernetes 风格文本输出账号名称、proxy 绑定、容器状态、代理环境摘要、DNS、默认路由、接口计数器、socket、监听进程、服务状态和近期网络错误分类。该命令不发上游请求或网络探针，不修改账号、proxy、runtime、调度或 YAML；机器读取仍需显式 `--json`。 + +## 建议 + +- 暂不因这次孤立 503 降权或摘除 `lyon9801`；最近 30 分钟为 37/37 成功。 +- 下一步源头优化优先研究 OpenAI response-header timeout 与客户 deadline/failover 预算的配合，使 5xx 在客户上下文取消前暴露；这属于配置候选，当前未实验、未下发。 +- 对 HTTP 200 后 `response.failed`，应继续作为 degraded 客户可见错误单列；响应已提交后不能依靠换号掩盖。 +- 若再次发生，直接用 request ID 同时抓 trace 与 infrastructure，并保留故障时刻的 Hysteria per-session 指标；不要用故障后的 socket 快照替代历史证据。 + +## 验证 + +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts` +- `bun --check scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts` +- `bun --check scripts/src/output.ts` +- `bun test scripts/src/output.test.ts`：2 pass，0 fail。 +- `git diff --check` +- `runtime.ts`：2970 行，低于 3000 行硬上限。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.11_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.11_Task_Report.md new file mode 100644 index 00000000..74c36ae1 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.11_Task_Report.md @@ -0,0 +1,68 @@ +# R2.2.11 任务报告 + +## 范围与口径 + +- 只读复取 PK01 Sub2API 最近2小时所有分组，并以最近30分钟作为趋势对照。 +- 请求分母、客户错误和上游错误以 Sub2API 原生 admin/ops 为主；system-log 缺失时才使用 target runtime 日志。 +- 分组与账号数据在滚动窗口内持续变化，报告使用本轮最后一次完整分组快照；共享账号的 upstream cause 不跨组相加。 +- 未发送探针，未修改 runtime、YAML、proxy、账号或调度配置。 + +## 所有分组概要 + +最近2小时初始总览：1391请求、37个客户错误，客户错误率2.66%；74个上游错误，上游错误率5.32%。滚动窗口下钻时请求数增长到约1403，但客户错误总数仍为37。 + +- `unidesk-codex-pool`：约763请求、26客户错误，错误率3.41%；上游错误率约7.86%。 +- `自用`：约640请求、11客户错误，错误率1.72%；上游错误率约2.19%。 +- `grok`：0请求、0错误。 +- 两个活跃组均无排队，查询时并发约9至10/218；没有证据把主因归为容量不足或需要降低并发。 +- 账号29 `https://api.lwylink.xyz pro 0.1` 与账号30 `https://api.lwylink.xyz plus 0.05` 因余额不足403不可用；其余7个OpenAI账号当前可用。 + +最近30分钟对照：412请求、19客户错误，错误率4.61%；27个上游错误，上游错误率6.55%。其中： + +- `unidesk-codex-pool`：251请求、17客户错误，错误率6.77%。 +- `自用`：160请求、2客户错误，错误率1.25%。 +- 错误没有整体消退，而是更集中到 `unidesk-codex-pool`。 + +## 主因一：账号27的Responses兼容性400 + +账号27 `https://sub.yjxm1221.top plus 0.025` 是当前客户错误主因： + +- 2小时：139次尝试、91成功、48失败，失败率34.53%，评分38/E；26个分组客户错误中占24个。 +- 最近30分钟：48次尝试、30成功、18失败，失败率37.50%；17个客户错误全部来自账号27且全部是400。 +- 2小时客户错误状态分布中400有21个，另有404、503、524；模型以 `gpt-5.6-terra`、`gpt-5.6-luna`、`gpt-5.6-sol` 为主。 +- 代表请求 `1f3fe800-a72a-4383-88ba-e7311abb8534` 在630 ms内失败，原文为 `Input must be a list`。 +- 代表请求 `ee991d2f-08fe-4251-a509-c3d1a4bd8e45` 先经历账号33的524和账号28的503，切到账号27后返回 `Invalid 'input[213].id'`，最终502。 +- 另有 `invalid_id_prefix` 类流式 `response.failed`。这些更符合该上游对Responses长历史/input item格式的兼容缺陷，不是并发、网络或普通5xx不稳定。 +- 通用400不能直接加入临时不可调度或failover，否则会把真实客户参数错误错误切号；若后续治理，应只针对已证实的精确兼容错误短语和未提交响应阶段处理。 + +## 主因二：长等待后504，切号已无时间窗 + +`自用`组11个客户错误全部是 `gpt-5.6-sol`：8个504、2个503、1个524，10个为流式。 + +- `lyon9801` 2小时109次尝试、100成功、9失败，失败率8.26%，评分75.2/C；8个客户错误，8次failover全部失败。 +- 请求 `02f0cc91-6f7f-4106-b902-49ee5260ed57` 等待约29分31秒后才收到账号15的504；1 ms后选号因 `context canceled` 失败。 +- 最近30分钟 `lyon9801` 已恢复为34/34成功、TTFT P95约7.8秒；窗口内2个客户错误均来自账号27，不是账号15。 +- 当前 OpenAI response-header timeout 为disabled。长等待是客户影响放大器，但现有数据只有端到端耗时，不能断言全部时间都在等待响应头，也不能直接把观测下限当配置值。 + +## 上游隔离与切号效果 + +- 账号28 `https://ai.whistlelads.com plus 0.03`：2小时24次尝试全部触发失败，但有26次临时不可调度事件，分组客户错误为0；大量错误被换号吸收。最近30分钟7次失败中5次切号恢复、2次失败。 +- 账号31 `https://sub2.pokexiao.com plus 0.03`：2小时215次尝试仅1次失败，评分93.1/A；最近30分钟126次中124成功，2次failover均恢复。 +- 账号33 `https://api.iceiu.com plus 0.03`：2小时约3.6%失败，评分B；最近30分钟失败率升到14.55%，并有6次HTTP 200后forward failure，需要关注。 +- 成功切号样本 `4d305f0f-3e4f-4e0d-86d1-060a03820d22`：账号32返回524、账号28返回503，约14秒后由账号27完成并最终HTTP 200，证明5xx failover链路能工作。 + +## HTTP 200后的客户体验错误 + +- `unidesk-codex-pool` 2小时有12个、最近30分钟有6个 `forwardFailureWithHttpSuccess`。 +- `自用` 2小时有3个、最近30分钟有1个同类事件。 +- 请求 `e41f8745-e223-4fbf-9fb8-041a40c8ad94` 最终HTTP 200，但约417秒后出现 `stream data interval timeout`，trace正确标记为degraded。 +- 这类请求不会进入普通HTTP客户错误分子，因此37个客户错误不是全部体验损失；分组间存在共享账号证据，不能简单把所有forward failure相加成全局唯一请求数。 + +## 调优建议 + +1. 优先治理账号27的精确Responses兼容错误族，而不是降低全池并发：先按错误短语、请求阶段和模型统计；候选方案是对已证实的 `Input must be a list`、`Invalid input[*].id`、`invalid_id_prefix` 在响应未提交前触发failover或暂时避免把对应请求形态路由到该账号。是否需要Sub2API源码变更取决于现有精确classifier能否配置表达，不能用通用400规则替代。 +2. 研究上游超时与客户deadline的预算关系：目标是让504或无响应在客户context取消前暴露并留出切号时间。缺少客户deadline和response-header阶段耗时前，不直接给出安全timeout数值。 +3. 保留账号28现有短冷却策略；它已显著避免客户错误，不建议仅因上游失败率高而长期禁用。 +4. 对账号33继续短窗口观察，重点看HTTP 200后 `response.failed`/stream timeout；这类错误无法在响应已提交后安全换号。 +5. 账号29、30的余额不足属于静态不可用，应由账号所有者补充余额或退役；当前无排队，暂不构成容量事故。 +6. 不因2小时旧错误降权 `lyon9801`；最近30分钟已恢复。若再次出现长等待504，按request ID同步抓trace和基础设施快照。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.12_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.12_Task_Report.md new file mode 100644 index 00000000..53ebf28f --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.12_Task_Report.md @@ -0,0 +1,66 @@ +# R2.2.12 任务报告 + +## 结果 + +已为 runtime-manual 账号 `https://sub.yjxm1221.top plus 0.025`（ID 27）下发精确 `Input must be a list` 临时不可调度规则： + +- statusCode：400 +- keyword：`input must be a list` +- duration：1分钟 +- 行为：命中后当前账号短时冷却，当前请求返回 `UpstreamFailoverError` 进入切号。 + +未使用通用400匹配，避免把真实客户参数错误错误切号。 + +## 所有权与配置 + +- owning YAML：`config/platform-infra/sub2api-codex-pool.yaml` +- 模板：`codex-upstream-failover` +- 账号管理类型：`runtime-manual` +- 下发方式：精准 `runtime apply --account 27 --template codex-upstream-failover --confirm` +- 未把账号27加入 YAML managed profiles，也未运行全量 `sync`。 + +为避免模板覆盖账号27既有运行规则，YAML模板同时对齐了其已有： + +- 429 `upstream rate limit exceeded` +- 504 `concurrency limit exceeded` / `upstream service temporarily unavailable` + +最终规则从6条增至7条，保留429、500、502、503、504、524，仅新增精确400。 + +## 源码确认 + +Sub2API v0.1.153 普通400进入 `handleErrorResponse` 后会调用 `handleOpenAIAccountUpstreamError`；`RateLimitService.HandleUpstreamError` 在默认400处理前执行 `tryTempUnschedulable`。规则命中返回 `shouldDisable=true`，OpenAI gateway 构造 `UpstreamFailoverError`，因此既冷却后续调度，也会为当前请求触发切号。 + +## 执行证据 + +Dry-run： + +- account：`https://sub.yjxm1221.top plus 0.025` / 27 +- change：update +- beforeRules：6 +- afterRules：7 +- beforeCodes：429,500,502,503,504,524 +- afterCodes：400,429,500,502,503,504,524 + +确认下发：`mutation=true`。最终 `runtime get`： + +- management：runtime-manual +- schedulable：true +- matching rule count：7 +- status codes：400,429,500,502,503,504,524 +- pool mode：false + +## 短窗口 + +下发后5分钟只读窗口： + +- `unidesk-codex-pool`：43请求、0客户错误、客户错误率0%。 +- 账号27：7次尝试、7次成功、0失败。 +- 窗口内没有自然出现新的 `Input must be a list`，因此尚未观察到真实 `account_temp_unschedulable` 命中；不能把“零命中”解释为规则无效。 + +## 验证 + +- 精准 runtime apply dry-run通过。 +- 精准 runtime apply confirm通过。 +- 逐账号 runtime get对账通过。 +- `git diff --check`通过。 +- 未发送错误探针，未修改proxy、账号凭据、priority、capacity、membership或其他调度配置。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.13_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.13_Task_Report.md new file mode 100644 index 00000000..29e9b61a --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.13_Task_Report.md @@ -0,0 +1,61 @@ +# R2.2.13 任务报告 + +## 下发结果 + +已向PK01当前全部非 `lyon9801` OpenAI账号精准批量下发 `codex-upstream-failover`：显式selector为19、27、28、29、30、31、32、33、34。 + +- selected：9 +- changed：8 +- succeeded：8 +- failed：0 +- noop：账号27已提前匹配模板 +- `lyon9801`（15）：未进入selector，保持0条规则、无模板 + +最终账号19、27、28、29、30、31、32、33、34均为7条规则，状态码集合400、429、500、502、503、504、524，模板均为 `codex-upstream-failover`。账号29、30仍保持余额不足导致的error与schedulable=false；模板下发未恢复账号状态。 + +## CLI修复 + +原CLI文档声明支持 `--accounts`，但parser实际拒绝。已恢复通用精准批量入口： + +- 支持1至100个逗号分隔ID或精确名称； +- `--account`与`--accounts`互斥； +- 仅apply/delete接受批量selector； +- 先解析全部selector并校验账号类型，再进行任何写入； +- 默认Kubernetes风格批量计划与结果表； +- 参数解析拆到 `runtime-options.ts`，`runtime.ts`收敛到2915行。 + +Artificer后续任务R2.2.14 / issue #1998负责把确认写入后的自动批量回读对账内建到CLI，消除外部逐账号get。 + +## 真实效果 + +下发后首个5分钟窗口： + +- 约85请求； +- 5个上游错误； +- 0客户错误； +- `unidesk-codex-pool`：5次临时不可调度、3次failover、3次全部恢复； +- `自用`：2次failover、2次全部恢复。 + +请求 `290215cf-e32b-44df-b2d6-842298c7e690` 证明新400规则真实命中：账号27收到上游400，规则1将其冷却1分钟，当前请求切到账号33，4.055秒后最终HTTP 200，错误未暴露给客户。 + +10分钟窗口出现的请求 `68cd320a-4d2b-482e-93ed-284625096d8f` 是模板下发前开始的旧长请求：等待约719秒后502，切号时context canceled，不能据此判定新模板失效。 + +## Skill固化 + +账号排障reference已固化： + +- 非auth错误优先使用状态码加稳定短语的通用模板； +- 先确认响应提交前能触发当前请求failover； +- 禁止账号、ID、provider URL或分组硬编码； +- 禁止通用400或宽泛invalid request； +- auth/token/余额/凭据状态保留专门路径。 + +## 验证 + +- 批量dry-run：9个selector全部解析，8 update、1 noop。 +- 批量confirm：8 succeeded、0 failed。 +- 再次批量dry-run：9/9 noop。 +- 逐账号运行态对账完成；后续由R2.2.14把该过程内建到CLI。 +- `bun --check`通过。 +- `git diff --check`通过。 +- skill quick validate通过。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.15_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.15_Task_Report.md new file mode 100644 index 00000000..f4945720 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.15_Task_Report.md @@ -0,0 +1,14 @@ +# R2.2.15 任务报告 + +Artificer 已完成只读调查并把完整证据写入 [GitHub Issue #1999 评论](https://github.com/pikasTech/unidesk/issues/1999#issuecomment-4966948330)。 + +核心结论： + +- 精确“状态码 + 稳定短语”模板适合吸收可确认的非 auth 上游兼容错误；不得使用 generic 400、generic invalid request 或账号/provider 特例。 +- 400 `Input must be a list` 已有真实切号成功证据。本轮新请求 `e5c08dde-9e60-4a14-9458-5574cec00b65` 进一步证明：首跳账号 `https://sub.yjxm1221.top plus 0.025` 的 400 可冷却并切号，但第二跳账号 `https://sub2.pokexiao.com plus 0.03` 等待约 60.6 秒后返回 504，第二次 failover 时 context 已取消。 +- 主代理已把同一精确短语加入通用 504 规则并下发非 `lyon9801` 账号；这只降低后续重复命中，不能缩短首次长等待。 +- `Expected an ID that begins with 'ctc'` 与 `invalid_id_prefix` 暂不下发，需先有跨账号成功切换证据，避免把客户载荷或会话亲和错误误判为上游故障。 +- 根治首次长等待穿透需要 Sub2API 源码能力：客户总 deadline、动态每跳首字/响应头预算及 failover reserve；静态 response-header timeout 或 Caddy timeout 不能完整替代。 +- 流式响应一旦向客户提交字节，Sub2API 不能安全重放，只能终止流并记录错误。 + +本子任务未发送探针，未修改 runtime、YAML、账号、proxy、调度或源码，无 commit、PR 和 rollout。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.16_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.16_Task_Report.md new file mode 100644 index 00000000..919aad77 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.16_Task_Report.md @@ -0,0 +1,23 @@ +# R2.2.16 任务报告 + +## 结论 + +用户 `mail0795336304@163.com` 的“卡住约 15 分钟”不发生在 Sub2API 的在途请求、账号排队或 proxy 3 出网阶段。服务端证据显示，上一笔请求于 2026-07-14 14:43:33（UTC+8）成功结束，下一笔直到 14:59:54 才进入 Sub2API，中间约 16 分 21 秒没有该用户的新入站请求。卡点因此位于 Sub2API 入站之前，优先怀疑客户端、本地 Agent 状态机或客户端到入口之间的请求提交链路。 + +## 原生运维证据 + +- 用户 ID：22；账号状态 active；用户并发上限 10，调查时 current concurrency 为 0。 +- API Key：名称 `pika_win`、ID 46、分组 `unidesk-codex-pool`；调查时 current concurrency 为 0。 +- 最近 30 分钟共 7 笔请求，均为成功，无用户可见错误。 +- 14:43:33 完成请求 `client:1daa3f3e-99a5-4fc4-809c-6f6ef854a2b2`，流式、`gpt-5.6-sol`，耗时 28.922 秒，账号 33。 +- 14:59:54 下一笔请求 `client:8d1f8ef9-e703-482c-ace9-0d8f13a5c8e3` 才进入服务端；内部 request ID `efdb251f-8e86-4f18-99ee-fa1094419dc0`，15:00:43 完成，HTTP 200，耗时 49.115 秒。 +- 最新请求使用上游账号 `https://api.iceiu.com plus 0.03`（ID 33）、模型 `gpt-5.6-sol`、proxy 3；没有 failover、select failure 或 forward failure。 +- proxy 3 为 `platform-infra-sub2api-pk01-local-egress-proxy`，127.0.0.1:10809；对应 Hysteria 客户端自 7 月 8 日持续 active，调查窗口无错误日志，当前存在正常连接。 + +## 判定边界 + +Sub2API v0.1.153 当前原生 request 明细记录请求完成耗时，但未记录流式首字时间和逐事件间隔。因此不能从现有数据精确证明客户端何时展示首字；不过“16 分 21 秒无入站请求”、用户/API Key 并发均为 0，以及随后请求 49 秒正常完成，足以排除 Sub2API 内部排队、上游账号占槽 15 分钟和 proxy 3 在该窗口报错。 + +## 建议 + +让用户侧保留该时段客户端日志，重点检查本地 Agent 是否在上一轮完成后未推进状态、请求是否直到 14:59:54 才真正发出，以及本地 HTTP/SSE 客户端是否错误等待已经关闭的流。后续 CLI 可泛化补充按用户邮箱映射用户 ID、请求间空窗和 TTFT/末事件可见性；本次只读调查未取消请求，也未修改 runtime、YAML、proxy 或调度。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.18_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.18_Task_Report.md new file mode 100644 index 00000000..f1bff722 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.18_Task_Report.md @@ -0,0 +1,26 @@ +# R2.2.18 任务报告 + +## 结果 + +PK01 Sub2API 已通过 YAML-first 受控流程从 v0.1.153 升级到 v0.1.155。实际运行镜像为 `docker.1panel.live/weishaw/sub2api:0.1.155`，容器状态 healthy。 + +## Source of truth + +- owning YAML：`config/platform-infra/sub2api.yaml` +- 仅修改 PK01 target 的 `image.tag`：`0.1.153` → `0.1.155` +- PR：https://github.com/pikasTech/unidesk/pull/2003 +- merge commit：`d97605d8f20d3c87ff16f747cd7e500d79f9827b` +- 未修改其他 target、公开域名、Secret、Codex pool、账号 runtime 或调度策略。 + +## 受控部署证据 + +- `platform-infra sub2api plan --target PK01`：通过。 +- `platform-infra sub2api apply --target PK01 --dry-run`：通过，渲染目标镜像 v0.1.155。 +- `platform-infra sub2api image-prepull --target PK01 --confirm --wait`：成功；镜像 ID `sha256:ec8c1452f9dc5f4ca...`。 +- apply job：`platform_infra_sub2api_apply_pk01_20260714074034649_757de2`，状态 succeeded，exit 0。 +- `platform-infra sub2api status --target PK01`：app running/healthy，实际镜像 v0.1.155；Redis保持原容器运行。 +- `platform-infra sub2api validate --target PK01`：通过；app、Redis、外部 PostgreSQL、本地 health 与 PK01 Caddy 公网 health 均成功。 + +## 边界 + +本次是纯镜像升级，没有执行 `codex-pool plan|sync|validate`，没有对齐或覆盖手工账号、临时不可调度规则、proxy 绑定和其他 runtime 配置。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.20_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.20_Task_Report.md new file mode 100644 index 00000000..266d79c9 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.20_Task_Report.md @@ -0,0 +1,42 @@ +# R2.2.20 任务报告 + +## 目标 + +快速缓解客户错误 #17440（request ID `e5c08dde-9e60-4a14-9458-5574cec00b65`），只使用 YAML-first 通用临时不可调度模板和受控 runtime CRUD，不修改 Sub2API 源码、账号容量、优先级、代理或凭据。 + +## 根因证据 + +- 原生 trace 显示请求先由账号 27（`https://sub.yjxm1221.top plus 0.025`）返回 400，精确短语 `Input must be a list` 已命中既有 400 规则并触发第一次 failover。 +- 第二跳账号 31（`https://sub2.pokexiao.com plus 0.03`）等待约 60.6 秒后返回 504；响应 detail 仍为 `Input must be a list`。 +- 第二次 failover 已进入，但选号时客户 context 已取消，最终 `reason=failover-attempted-no-candidate`。因此客户可见 504 的直接原因是第二跳耗尽请求时间窗，而不是首跳 400 未切号。 +- 账号 31 在此前 30 分钟有 75 次尝试、8 次失败、失败率 10.67%，该窗口唯一客户错误就是本请求；2 小时窗口质量仍为 B，故不据单次事件调整长期容量或优先级。 + +## 配置调优 + +owning YAML：`config/platform-infra/sub2api-codex-pool.yaml`。 + +在通用模板 `codex-upstream-failover` 的精确 504 规则中加入关键词 `input must be a list`，冷却时间保持 1 分钟。该规则只匹配实际 504 与稳定短语，不泛化到其他 4xx/5xx，也不涉及 auth 错误。 + +通过显式 selector `19,27,28,29,30,31,32,33,34` 对 9 个非 `lyon9801` OpenAI 账号执行： + +1. 批量 dry-run：selected=9、changed=9、failed=0。 +2. 批量 confirm：selected=9、succeeded=9、failed=0。 +3. 回读账号 31 和 27：504 规则均包含 `input must be a list`，模板匹配正常。 +4. 回读 `lyon9801`：临时不可调度仍为 disabled、0 条规则，例外未被改变。 + +## 下发后观察 + +下发后的近期 10 分钟原生 Ops 窗口： + +- 请求 88； +- 客户错误 0，错误率 0.00%； +- failover 8 次，8 次均恢复成功； +- select failed 0； +- 临时不可调度命中 4 次； +- 当前排队 0。 + +该窗口尚未出现新的“504 + Input must be a list”样本，因此只能证明配置已实际生效且未见副作用，不能把短窗口零错误宣称为长期因果验证。 + +## 边界与剩余风险 + +本次规则会在同类 504 首次发生后短时摘除对应账号，减少连续客户请求重复命中；它无法挽救已经在第二跳耗尽 context 的当前请求。若同类首次错误仍频繁穿透，下一步应从 Sub2API 源码能力解决 failover 时间预算/每跳超时，而不是继续扩大关键词或盲目降低全池并发。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.22_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.22_Task_Report.md new file mode 100644 index 00000000..ddf9d433 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.22_Task_Report.md @@ -0,0 +1,59 @@ +# R2.2.22 任务报告 + +## 结论 + +错误 #17490（request ID `4eb9a84b-7346-47a8-bb66-fb7b44317279`）已确认由 Sub2API 内部渠道监控使用的 `monitor` API Key（ID 19）发起。账号 `https://sub2.pokexiao.com plus 0.03`（ID 31）长等待后返回 502；内部监控客户端已因响应头超时取消请求，Sub2API 却继续等待被去取消的上游请求。上游错误返回后虽然进入 failover，下一轮选号使用的原始 request context 已取消，无法开始第二跳，最终记录 502。 + +## 原生 trace + +- 请求：`/v1/responses`，`gpt-5.6-sol`，同步。 +- 调用方：用户 1、API Key `monitor`（ID 19）。 +- 单跳总耗时约 54,339 ms。 +- 上游状态：502，原始 detail：`error code: 502`。 +- Sub2API 记录 `openai.upstream_failover_switching`，switch 1/10。 +- 同一毫秒选号失败：`context canceled`，排除账号数 1。 +- 最终 HTTP 502，`reason=failover-attempted-no-candidate`。 + +近 30 分钟另一条客户错误 #17491（request ID `426551fe-cdd4-4563-a3d0-89b97aa5ffa3`）完全同型，但来自不同用户/IP，不是内部 monitor：同一账号和原始 502 detail，流式，单跳约 36,157 ms，failover 后立即 `context canceled`。因此内部监控 timeout 能解释 #17490，但不能解释所有账号 31 的客户穿透。 + +## 内部监控 timeout 与源码链 + +Sub2API v0.1.155 官方源码中，渠道监控 timeout 是 MVP 硬编码值： + +- `monitorResponseHeaderTimeout = 30s`：等待响应头； +- `monitorRequestTimeout = 45s`：单次模型请求总超时； +- runner 外层使用请求超时、PING 超时和缓冲构造总 context。 + +网关上游转发则调用 `detachUpstreamContext(ctx)`，其实现为 `context.WithoutCancel(ctx)`；failover 选号继续使用 `c.Request.Context()`。因此 #17490 的完整时间线是： + +1. 内部渠道监控发起请求； +2. 约 30 秒仍无响应头，监控 HTTP client 取消请求并记录 timeout； +3. 网关上游请求忽略该取消，继续等待账号 31； +4. 约 54.3 秒账号 31 返回 502； +5. handler 触发 failover，但原始监控 request context 已取消； +6. 选号立即 `context canceled`，没有第二跳。 + +内部监控的 30 秒响应头超时确实等不到本次切号，但根因不仅是 timeout 数值短，还包括“首轮上游去取消、后续选号绑定已取消 context”的生命周期不一致。 + +## 账号与策略 + +账号 31 调查窗口内 163 次尝试、149 次成功、14 次失败，失败率 8.59%，质量 D/69；14 次 failover 信号中 2 次客户穿透。现有模板未命中该账号的 502，临时不可调度计数为 0。 + +原因是规则匹配原始上游响应体。模板已有 `upstream service temporarily unavailable` 等关键词，但原始 body 只有 `error code: 502`；UI 包装消息不是规则匹配输入。 + +## 基础设施 + +账号 31 绑定 proxy 3：`platform-infra-sub2api-pk01-local-egress-proxy`，HTTP `127.0.0.1:10809`；owner `host-proxy-hysteria.service` active/running、0 重启。Sub2API app 0 restart、无 OOM，主机接口无 RX/TX error/drop。同一 proxy 的其他账号仍有成功请求。查询时快照不能排除历史抖动，但没有本地 proxy 整体故障证据。 + +## 调优候选 + +1. 内部监控：把响应头 timeout、总请求 timeout 和 cadence 从硬编码提升为原生可配置事实，并在 CLI 展示来源；timeout 必须与网关每跳 deadline/failover reserve 联合设计，不能只单独放大。 +2. 网关源码：统一上游与 failover 生命周期，记录 client deadline、hop deadline、remaining budget 和 cancel source。客户端断开后若无需继续工作，应取消首轮上游；若要内部完成 failover，则选号与下一跳必须使用有界内部 context，同时停止无意义的下游写回。 +3. 配置缓解：精确短语 `error code: 502` 可进入通用非 auth 502 临时不可调度模板，减少后续请求再次命中，但不能挽救首次请求。 +4. 不建议仅降低全池并发；窗口无排队，故障发生在单账号长等待。 + +本任务只读，未修改 runtime、YAML、账号、proxy、调度或 Sub2API 源码。 + +## 原生记录闭环 + +channel 3 记录 15589：checkedAt 2026-07-14T08:52:45Z，latency 30,011 ms，错误为 http2: timeout awaiting response headers；对应 #17490 从约 08:52:45Z 开始、网关在 08:53:39Z 结束，证明内部监控在 30 秒先取消，而去取消的首轮上游继续约 24 秒后才返回并尝试 failover。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.24_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.24_Task_Report.md new file mode 100644 index 00000000..073a9057 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.24_Task_Report.md @@ -0,0 +1,47 @@ +# R2.2.24 任务报告 + +## 结论 + +错误 #17511（请求 `f7ff0efe-64d3-4dd7-8716-20bdfc3a2f90`）不是内部 monitor 请求，也不是账号池没有候选。它由用户 1 的 `main` API Key（ID 20）发起，来源 IP `152.53.229.148`，分组“自用”（ID 3），流式 `/responses`。 + +账号 `https://sub2.pokexiao.com plus 0.03`（ID 31）等待 63,238 ms 后返回上游 502；Sub2API 在 1 ms 内记录 `openai.upstream_failover_switching`，随后选号立即以 `context canceled` 失败，最终 63,239 ms 返回 502。因此准确表述是：切号已触发，但切号开始时请求 context 已取消，备用账号没有被选取和调用。 + +同一 Key、同一来源 IP 的请求 `0503e439-ee9b-4954-bc2d-3087ec3a1d5e` 在此前 6 分钟出现相同链路：账号 31 等待 75,965 ms 后返回 504，切号同毫秒触发，选号立即 `context canceled`。这证明不是单次调度抖动。 + +## 运行面证据 + +- “自用”分组在窗口内有 9 个账号，其中 7 个可用，当前 18/218 并发，队列 0。 +- #17511 选号失败时仅排除账号 31，不存在“所有备用账号均被排除”。 +- 账号 31 最近窗口 286 次尝试、277 成功、9 失败，失败率 3.15%，质量分 89/B；9 次 failover 中 2 次最终客户错误。 +- OpenAI 专用响应头超时当前为 disabled，来源为进程环境。 +- 成功流式请求 TTFT P99 约 61.6 秒、最大约 92.9 秒；因此不能未经验证直接把全局响应头超时压到 45–60 秒。 + +## v0.1.155 源码归因 + +`openai_gateway_handler.go` 的 failover 循环： + +1. 选号使用 `c.Request.Context()`。 +2. 上游转发进入 `Forward(c.Request.Context(), ...)`。 +3. `Forward` 构建上游请求前调用 `detachUpstreamContext(ctx)`。 +4. `detachUpstreamContext` 返回 `context.WithoutCancel(ctx)`，官方测试 `TestDetachUpstreamContextIgnoresClientCancel` 明确验证客户端取消不会取消上游请求。 +5. 首轮上游最终返回可 failover 的 502/504 后，handler 回到循环，再次用原始且已取消的 `c.Request.Context()` 选号，于是立即失败。 + +这是“首轮上游去取消、后续选号仍绑定下游取消”的生命周期不一致。它允许客户端断开后首轮上游继续消耗时间，但不保留后续 failover 的执行窗口。 + +## 版本与配置判断 + +- v0.1.153、v0.1.152、v0.1.151、v0.1.150、v0.1.149 均存在相同 `context.WithoutCancel` 实现及调用，回退到这些版本不能修复该问题。 +- 公网 Caddy owning YAML 的 `responseHeaderTimeoutSeconds` 为 600 秒，明显长于本次 63 秒，不支持“Caddy 先截断”的假设。 +- 简单降低全局 `gateway.openai_response_header_timeout` 能让首轮更早失败并为切号留预算，但当前成功 TTFT 长尾已超过 60 秒，未经按请求类型/账号评估会误杀有效请求。 +- 精确匹配账号 31 的原始错误短语 `error code: 502` 并临时不可调度可减少后续请求再次命中，但不能挽救触发规则的首个请求，也不能修复 context 生命周期。 + +## 建议 + +1. 源码根治：统一上游请求与 failover 循环的生命周期。至少在进入下一轮前检测原始 context；若业务要求客户端断开后停止工作，则不要对上游使用 `WithoutCancel`。若业务要求内部继续完成 failover，则选号、转发和最终写回必须使用一个有界的内部 deadline，并在客户端已断开时停止无意义的继续写回。 +2. 增加 failover reserve：每次首字节等待不能耗尽整体预算，应根据请求类型（普通/compact）和剩余预算设置 per-hop deadline；日志需记录 client deadline、hop deadline、remaining budget 和 cancel source。 +3. 配置缓解前先用 CLI 按普通/compact、账号统计 TTFT 与错误等待时长，再选择高于正常长尾且低于客户端 deadline 的阈值；当前证据不足以安全给出单一秒数。 +4. 对账号 31 的稳定原始 502 短语使用通用非 auth 临时不可调度模板，只作为减少连续命中的缓解，不宣称根治。 + +## 官方 main 核对 + +2026-07-14 只读抓取官方 Wei-Shaw/sub2api main 后，detachUpstreamContext 仍返回 context.WithoutCancel(ctx)，Responses failover 选号仍使用 c.Request.Context()；官方最新主线尚未修复该生命周期不一致。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.8_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.8_Task_Report.md new file mode 100644 index 00000000..92c5cb27 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.8_Task_Report.md @@ -0,0 +1,55 @@ +# R2.2.8 任务报告 + +## 结果 + +已增强 Sub2API 只读诊断 CLI，按分组和上游账号归因近期请求质量。默认输出保持 Kubernetes 风格表格，完整展示账号名称和账号 ID；显式 `--json` 提供紧凑语义摘要，整组概要通过 `--account` 和 request ID 渐进下钻。未修改 Sub2API runtime、YAML、账号调度或源码运行面。 + +## 数据口径 + +- 成功请求、流式请求和账号 TTFT 来自 Sub2API 原生 `/api/v1/admin/usage`。 +- 客户可见错误来自原生 `/api/v1/admin/ops/request-errors`。 +- 切号、同账号重试、临时不可调度和 `forward_failed` 来自原生 system-log 索引，并自动分页读取近期窗口。 +- 失败分子按 request ID 去重；账号 attempts 是上游账号尝试次数，不等于客户请求数，切号时一个客户请求可以计入多个账号。 +- 评分权重为可靠性 60、TTFT 25、当前可用性 15，置信度单独计算。TTFT 少于 5 个样本不按零时延计分。 +- 至少 10 次尝试且至少 5 个 TTFT 样本时，才把高分映射为 A-D 可比较等级；样本不足的偶然高分标记为 `insufficient`。至少 10 次尝试且评分低于 60 时保留 E，避免高失败率被证据不足掩盖。 +- 账号名称是人类可读运维信息，报告和 CLI 完整输出；API Key、token、credentials 继续脱敏。 + +## 近期只读快照 + +采样窗口为 2026-07-14 05:32-05:35 UTC 向前 60 分钟，滚动值会随请求变化。 + +- 所有分组共 530 个请求、13 个客户错误，客户错误率 2.45%，上游错误率 6.60%。 +- `unidesk-codex-pool`：约 219 个请求、7 个客户错误，客户错误率 3.20%，上游错误率 13.24%，队列为 0。 + - `https://api.iceiu.com plus 0.03`（ID 33）：89.9/B/healthy，高置信；172 次尝试、3 次失败，失败率 1.74%，TTFT P95 42.87 秒；有 3 个 `forward_failed`，未形成客户错误。 + - `https://sub.yjxm1221.top plus 0.025`（ID 27）：15/E/poor，中置信；39 次尝试、26 次失败，失败率 66.67%，TTFT P95 202.12 秒；23 个切号请求，18 个恢复、5 个失败，3 次临时不可调度、3 个 `forward_failed`、7 个客户错误。它是该分组近期客户可见错误的主因。 + - `https://ai.whistlelads.com plus 0.03`（ID 28）：20/E/poor，低置信；11 次尝试全部失败；11 个切号请求全部由后续账号恢复，12 次临时不可调度，未形成客户错误。说明现有切号有效遮蔽了该账号故障，但源账号质量差。 + - `https://sub2.pokexiao.com plus 0.03`（ID 31）：样本内 100 分，但只有 8 次尝试，等级为 `insufficient`，不能据此认定为优质账号。 + - `https://api.lwylink.xyz pro 0.1`（ID 29）和 `https://api.lwylink.xyz plus 0.05`（ID 30）当前不可用且无近期成功证据。 +- `自用`：约 326 个请求、5 个客户错误，客户错误率 1.53%，上游错误率 2.15%，队列为 0。 + - `https://api.iceiu.com plus 0.03`（ID 33）：93.5/A/preferred，高置信；230 次尝试全部成功，TTFT P95 53.94 秒。 + - `https://sub2.pokexiao.com pro 0.05`（ID 32）：77.9/C/watch，中置信；33 次尝试、1 次失败，切号后恢复，TTFT P95 98.49 秒。 + - `lyon9801`（ID 15）：55.9/E/poor，中置信；34 次尝试、5 次失败，失败率 14.71%；5 次切号均未恢复，5 个客户错误，是该分组近期客户可见错误主因。 +- `grok` 分组近期无请求和错误。 +- 两个活跃分组均无排队，当前证据不支持把降低全局上游并发作为主修复方向。 +- 近期未观察到同账号重试事件；不能把“未观察到”解释为功能不存在。 + +## CLI 行为 + +- `runtime errors --all-groups`：所有分组概要和稳定分页。 +- `runtime errors --group `：分组内完整账号名称、评分、置信度、错误、切号和 TTFT 表格。 +- `runtime errors --group --account `：单账号详细归因和 request index。 +- 整组 `--json` 保留所有账号名称和质量概要；请求级证据通过 `--account` 下钻，不依赖手工 `limit`。 +- 单账号 JSON 实测 9248 字节、整组 JSON 实测 9058 字节，均低于 10240 字节输出预算；未出现 `outputTruncated`，也未暴露内部 `_requestIds`。 + +## 结论与后续建议 + +主因不是池总并发或排队，而是少数上游账号的高失败率和长 TTFT：`https://sub.yjxm1221.top plus 0.025` 与 `lyon9801` 会把错误直接传给客户，`https://ai.whistlelads.com plus 0.03` 虽被切号遮蔽但持续消耗 failover。下一步若获准调优，应优先基于高置信/中置信评分调整账号优先级或临时不可调度，并继续保持 `lyon9801` 的既有例外约束；低置信账号先积累样本，不应自动调度。任何自动评分到 runtime 的闭环都应另行审批，本任务没有实施。 + +## 验证 + +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts` +- `bun --check scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts` +- `bun --check scripts/src/output.ts` +- `bun test scripts/src/output.test.ts`：2 pass，0 fail。 +- `git diff --check` +- PK01 原生只读命令已验证 all-groups、group 2、group 3、account 27、默认表格和显式 JSON。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.9_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.9_Task_Report.md new file mode 100644 index 00000000..6705c7d9 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.9_Task_Report.md @@ -0,0 +1,48 @@ +# R2.2.9 任务报告 + +## 范围 + +只读检查 PK01 Sub2API 最近 30 分钟的所有分组、账号质量、客户可见错误、切号结果、临时不可调度和代表请求链路。未修改 runtime、YAML、账号调度或 Sub2API 运行面。 + +## 全局快照 + +抓取时间为 2026-07-14 06:14-06:15 UTC，窗口向前 30 分钟，数据会随滚动窗口变化。 + +- 所有分组共 440 个请求，7 个客户可见错误，客户错误率 1.59%；13 个上游错误，上游错误率 2.95%。 +- `unidesk-codex-pool`：约 257-258 个请求，4 个客户错误，错误率约 1.56%，上游错误率约 3.89%。 +- `自用`：约 182-183 个请求，3 个客户错误，错误率约 1.64%，上游错误率约 1.64%。 +- `grok`：无请求、无错误。 +- 两个活跃分组查询时并发约 8-9/218，排队为 0；当前证据不支持把降低池总并发作为主修复。 +- 当前有 7/9 个账号可用。`https://api.lwylink.xyz pro 0.1`（ID 29）和 `https://api.lwylink.xyz plus 0.05`（ID 30）因余额不足 403 处于错误状态。 + +## unidesk-codex-pool + +- `https://sub2.pokexiao.com plus 0.03`（ID 31）：95.6/A/preferred，高置信；84 次尝试全部成功，TTFT P95 39.95 秒。 +- `https://api.iceiu.com plus 0.03`（ID 33）：87.9/B/healthy，高置信；90 次尝试、2 次失败，失败率 2.22%，TTFT P95 46.90 秒；产生 2 个客户错误。 +- `https://sub.yjxm1221.top plus 0.025`（ID 27）：66.2/D/degraded，高置信；55 次尝试、6 次失败，失败率 10.91%，TTFT P95 17.56 秒；产生 2 个客户错误、4 个 `forward_failed`。其 2 次切号均恢复。 +- `https://ai.whistlelads.com plus 0.03`（ID 28）：7 次尝试全部失败，证据不足等级；7 次切号全部恢复、7 次临时不可调度，没有形成客户错误，说明内部 failover 有效遮蔽了该账号近期故障。 +- 该分组 4 个客户错误均为同步请求：2 个 400、2 个 404，模型为 `gpt-5.6-luna` 和 `gpt-5.6-sol`。 +- 分组共 8 次切号，8 次全部恢复；没有候选耗尽。 +- request `46bdd23b-e8bf-461d-bb3f-ad6313412ae3`：账号 `https://sub.yjxm1221.top plus 0.025`，最终 502；上游原文是 `400 Input must be a list`，总耗时约 977 秒。该错误更像请求结构或协议兼容错误，不应按临时上游过载盲目重试。 +- request `bac355f9-22fc-4d18-a899-8f609d847615`：账号 `https://sub.yjxm1221.top plus 0.025`，HTTP 200 后出现 `stream usage incomplete: missing terminal event`，trace 判为 degraded，说明成功状态不能代表客户收到完整流式响应。 + +## 自用 + +- `https://sub2.pokexiao.com plus 0.03`（ID 31）：99.1/A/preferred，中置信；23 次尝试全部成功，TTFT P95 15.98 秒。 +- `https://sub.yjxm1221.top plus 0.025`（ID 27）：98.5/A/preferred，高置信；55 次尝试全部成功，TTFT P95 19.94 秒。同一账号在不同分组的模型和请求构成不同，不能跨分组合并评分。 +- `https://api.iceiu.com plus 0.03`（ID 33）：86.7/B/healthy，中置信；39 次尝试、1 次失败，产生 1 个 524 客户错误；1 次切号失败。 +- `lyon9801`（ID 15）：81.6/B/healthy，中置信；49 次尝试、3 次失败，失败率 6.12%，TTFT P95 9.89 秒；产生 2 个 503 流式客户错误。虽然综合评分为 B，但它仍是该分组近期客户错误主因。 +- 分组共 3 次切号，全部失败并出现 3 次选号失败。 +- request `da415526-b124-4e23-a4a7-ea48154c851e`：账号 `lyon9801`，流式 `gpt-5.6-sol`；约 1556 秒后才收到 503 并触发切号，此时 context 已取消，选号在约 1 毫秒内失败，trace 判为 `failover-attempted-no-candidate`。这是 failover 时机过晚，不是池容量不足。 +- request `4bba7f9b-15e3-4891-b063-bd65fb6e3704`：账号 `lyon9801`，HTTP 200 后发生 `upstream response failed`，trace 判为 degraded，客户可能收到不完整流。 + +## 判断 + +- 相比上一轮 60 分钟快照，当前短窗口的客户错误率和上游错误率数值更低，但窗口不同且持续滚动，不能直接解释为配置调优效果。 +- 当前主要客户可见错误已经从纯上游过载，分化为三类： + - 请求/协议兼容错误：`Input must be a list`、404； + - 超长等待后才 failover，客户上下文已取消； + - HTTP 200 已提交后缺少终止事件或后续流式失败。 +- 对第一类不应无差别重试；对第二类应继续调查 response-header/首事件等待和 failover 触发时机；对第三类需要把 degraded 流式结果纳入客户错误治理，不能仅看 HTTP 状态。 +- 最近 30 分钟未观察到同账号重试事件；不能把零事件解释为重试能力不存在。 +- 当前只完成分析，没有下发任何调优。 diff --git a/docs/MDTODO/sub2api-upstream-reliability.md b/docs/MDTODO/sub2api-upstream-reliability.md index 3a3c36ba..27bc7422 100644 --- a/docs/MDTODO/sub2api-upstream-reliability.md +++ b/docs/MDTODO/sub2api-upstream-reliability.md @@ -41,6 +41,60 @@ #### R2.2.7 按仓库 P0 拆分超过 3000 行的 `remote-python-sync.ts`：把大段内嵌 Python 按 trace、sync/validate 与 sentinel 职责迁入原生模块，单文件收敛到 2000 行以下，并保持受控 CLI 行为与只读诊断契约不变，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.7_Task_Report.md)。 +#### R2.2.8 [completed] + +增强 Sub2API CLI 的上游账号粒度质量分析与自动评分：使用原生 Ops 归因账号请求量、客户/上游错误、触发切号、切号结果、同账号重试、临时不可调度和首字延迟覆盖率/分位数，输出可解释评分、置信度和扣分原因；只做分析，不自动修改账号 runtime 或调度配置，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.8_Task_Report.md)。 +#### R2.2.9 [completed] + +重新抓取 PK01 Sub2API 最近 30 分钟所有分组的客户可见错误、上游错误、账号质量、切号恢复、临时不可调度和首字延迟，按完整账号名称报告主因；只做原生只读分析，不修改 runtime、YAML 或调度配置，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.9_Task_Report.md)。 +#### R2.2.10 [completed] + +只读专项调查 lyon9801 的近期 503 流式错误：增强 Sub2API CLI 同时采集账号 proxy 绑定、容器环境代理摘要、DNS/路由、连接与出网基础设施证据，结合真实 request trace 和源码链路区分 OpenAI 上游、PK01 网络、代理、超时或 Sub2API 转发问题；不发探针、不修改 runtime、YAML、代理或调度配置，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.10_Task_Report.md)。 +#### R2.2.11 [completed] + +只读复取并综合分析最近2小时所有 Sub2API 分组的客户可见错误、上游错误、账号质量、failover、候选耗尽和流式 degraded 情况，报告完整账号名称，不发探针且不修改 runtime、YAML、代理或调度配置，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.11_Task_Report.md)。 +#### R2.2.12 [completed] + +快速缓解账号 https://sub.yjxm1221.top plus 0.025 的 Input must be a list 客户可见400：在 owning YAML 模板加入精确400短语并保留该 runtime-manual 账号现有规则，通过精准 runtime CRUD dry-run、确认下发和短窗口只读复取验证当前请求能够冷却并切号，不泛化匹配其他400，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.12_Task_Report.md)。 +#### R2.2.13 [completed] + +向当前PK01全部非 lyon9801 OpenAI账号精准批量下发 codex-upstream-failover 统一临时不可调度模板：从runtime账号清单固定显式selector，排除lyon9801，先dry-run再confirm并逐账号对账，不修改分组、优先级、容量、proxy或凭据，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.13_Task_Report.md)。 +#### R2.2.14 [in_progress] + +由 Artificer 实现 Sub2API runtime 精准批量 CRUD 的命令内自动回读对账：confirm 后批量回读全部selector并输出write result、actual、reconciled、mismatch和部分写入摘要，避免外部逐账号get；关联 [GitHub issue #1998](https://github.com/pikasTech/unidesk/issues/1998)，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.14_Task_Report.md)。 +#### R2.2.15 [completed] + +继续只读分析统一非 auth 上游模板与超时预算的下一轮调优方案：交由独立 Artificer 调查 [Issue #1999](https://github.com/pikasTech/unidesk/issues/1999)，基于下发后真实400切号成功、旧长请求context canceled、其他invalid input id与HTTP 200流式degraded证据，区分可安全泛化的精确模板、需要Sub2API源码能力和不应自动处理的客户参数错误，不继续盲目扩规则，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.15_Task_Report.md)。 + +#### R2.2.16 [completed] + +紧急只读追踪用户 mail0795336304@163.com 反馈请求卡住15分钟：定位用户ID、近30分钟请求、当前长请求/并发、request ID、账号与模型，区分排队、上游首字、流式间隔、代理网络或客户连接，不取消请求且不修改runtime、YAML、proxy或调度，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.16_Task_Report.md)。 +#### R2.2.17 [in_progress] + +复盘用户直接反馈的只读追踪过程并交由 Artificer 实现 [Issue #2000](https://github.com/pikasTech/unidesk/issues/2000)：增加通用的一键用户追踪 CLI，以邮箱或用户ID关联请求空窗、用户/API Key并发、client/internal request ID、上游账号名称、proxy基础设施和证据化归因，采用Kubernetes列表风格与稳定ID渐进披露，减少手工API与重复trace调用，不改运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.17_Task_Report.md)。 +#### R2.2.18 [completed] + +将 PK01 的 Sub2API 通过 owning YAML 从 v0.1.153 受控升级到 v0.1.155：只修改 PK01 target镜像版本，依次执行状态核对、plan、apply dry-run、镜像预拉、受控apply、状态与公开健康验证，保持账号池runtime和Codex pool配置原样，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.18_Task_Report.md)。 +#### R2.2.19 [in_progress] + +交由 Artificer 实现 [Issue #2004](https://github.com/pikasTech/unidesk/issues/2004)：增强 Sub2API CLI 拉取 v0.1.155 原生智能诊断与渠道监控状态，泛化所有分组/渠道概要和指定对象下钻，以Kubernetes列表风格呈现TTFT、上游与客户错误率、原生建议和证据边界，以及7/15/30天渠道运行状态、平台/模型、对话延迟、端点PING、可用率、最近记录和刷新事实，使用稳定ID渐进披露，不重算第二套诊断且不修改运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.19_Task_Report.md)。 + +#### R2.2.20 [completed] + +快速缓解错误 #17440：基于 request e5c08dde-9e60-4a14-9458-5574cec00b65 的真实链路，将精确短语 Input must be a list 补入通用 504 临时不可调度规则，对全部非 lyon9801 账号精准批量 dry-run、确认下发和自动回读对账，不改容量、优先级、代理或 Sub2API 源码，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.20_Task_Report.md)。 +#### R2.2.21 [in_progress] + +交由 Artificer 实现 [Issue #2013](https://github.com/pikasTech/unidesk/issues/2013)：定义并实现 Sub2API P0/P1/P2 故障快速查询 CLI，P0 为上游故障穿透客户，P1 为首字延迟偏高且普通与 compact 分开统计并突出极端离群，P2 为上游错误率偏高；支持所有分组概要与按等级、分组、账号名称、模型、请求类型、稳定 request ID 下钻，默认 Kubernetes 列表风格和固定分页，不靠手工 limit，不修改运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.21_Task_Report.md)。 + +#### R2.2.22 [completed] + +主代理只读追踪客户错误 #17490（request ID 4eb9a84b-7346-47a8-bb66-fb7b44317279）：使用 Sub2API 原生 Ops、trace、账号完整名称、failover/retry/临时不可调度和近期窗口证据，定位账号 https://sub2.pokexiao.com plus 0.03 的同步502为何穿透客户，区分候选耗尽、context取消、不可重试阶段或模板未命中；先调查，不盲目新增generic 502规则，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.22_Task_Report.md)。 +#### R2.2.23 [in_progress] + +交由 Artificer 实现 [Issue #2016](https://github.com/pikasTech/unidesk/issues/2016)：增强 CLI 对 Sub2API 内部账号/渠道监控的原生只读访问，展示探测 timeout、cadence、阶段耗时、账号选择、上游首字节、切号/重试和最终状态，支持所有监控对象概要与稳定 ID 下钻，默认 Kubernetes 列表风格且不靠手工 limit；不修改运行面、不发新探针，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.23_Task_Report.md)。 +#### R2.2.24 [completed] + +只读调查错误 #17511（request ID `f7ff0efe-64d3-4dd7-8716-20bdfc3a2f90`）为何上游 502 后未成功切号，重建流式请求的账号选择、首字节、响应提交、failover、排除列表、context deadline 与最终客户错误链，并与内部 monitor 请求 #17490 对照，不修改运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.24_Task_Report.md)。 ### R2.3 调查并收敛 Sub2API 原生 system-log 索引未保留策略事件的能力缺口，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.3_Task_Report.md)。 diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-options.ts b/scripts/src/platform-infra-sub2api-codex/runtime-options.ts new file mode 100644 index 00000000..da45d5dd --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/runtime-options.ts @@ -0,0 +1,120 @@ +import { defaultCodexPoolRuntimeTargetId } from "./runtime-target"; + +export type RuntimeAction = "list" | "get" | "errors" | "infrastructure" | "apply" | "delete"; + +export interface RuntimeOptions { + action: RuntimeAction; + account: string | null; + accounts: string[]; + group: string | null; + allGroups: boolean; + platform: string | null; + pageToken: string | null; + template: string | null; + kind: "temp-unschedulable"; + confirm: boolean; + full: boolean; + raw: boolean; + json: boolean; + since: string; + tail: number; + targetId: string; +} + +export function parseRuntimeOptions(args: string[]): RuntimeOptions { + const [actionRaw = "list", ...rest] = args; + if (!(["list", "get", "errors", "infrastructure", "apply", "delete"] as string[]).includes(actionRaw)) { + throw new Error("runtime usage: list|get|errors|infrastructure|apply|delete [--account |--accounts ] [--group |--all-groups] [--platform ] [--page-token ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--json|--full|--raw]"); + } + let account: string | null = null; + let accounts: string[] = []; + let group: string | null = null; + let allGroups = false; + let platform: string | null = null; + let pageToken: string | null = null; + let template: string | null = null; + let kind = "temp-unschedulable" as const; + let confirm = false; + let full = false; + let raw = false; + let json = false; + let since = "24h"; + let tail = 50_000; + let targetId = defaultCodexPoolRuntimeTargetId(); + for (let index = 0; index < rest.length; index += 1) { + const arg = rest[index]!; + const readValue = (name: string): string => { + const value = rest[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); + index += 1; + return value.trim(); + }; + if (arg === "--confirm") confirm = true; + else if (arg === "--full") full = true; + else if (arg === "--raw") raw = true; + else if (arg === "--json") json = true; + else if (arg === "-o" && readValue("-o") === "json") json = true; + else if (arg === "--account") account = readValue("--account"); + else if (arg.startsWith("--account=")) account = arg.slice("--account=".length).trim(); + else if (arg === "--accounts") accounts = parseAccountSelectors(readValue("--accounts")); + else if (arg.startsWith("--accounts=")) accounts = parseAccountSelectors(arg.slice("--accounts=".length)); + else if (arg === "--group") group = readValue("--group"); + else if (arg.startsWith("--group=")) group = arg.slice("--group=".length).trim(); + else if (arg === "--all-groups") allGroups = true; + else if (arg === "--platform") platform = readValue("--platform"); + else if (arg.startsWith("--platform=")) platform = arg.slice("--platform=".length).trim(); + else if (arg === "--page-token") pageToken = readValue("--page-token"); + else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim(); + else if (arg === "--template") template = readValue("--template"); + else if (arg.startsWith("--template=")) template = arg.slice("--template=".length).trim(); + else if (arg === "--kind") kind = parseRuntimeKind(readValue("--kind")); + else if (arg.startsWith("--kind=")) kind = parseRuntimeKind(arg.slice("--kind=".length)); + else if (arg === "--target") targetId = readValue("--target"); + else if (arg.startsWith("--target=")) targetId = arg.slice("--target=".length).trim(); + else if (arg === "--since") since = readValue("--since"); + else if (arg.startsWith("--since=")) since = arg.slice("--since=".length).trim(); + else if (arg === "--tail") tail = parseRuntimeTail(readValue("--tail")); + else if (arg.startsWith("--tail=")) tail = parseRuntimeTail(arg.slice("--tail=".length)); + else throw new Error(`unsupported runtime option: ${arg}`); + } + const action = actionRaw as RuntimeAction; + if (account !== null && accounts.length > 0) throw new Error("use only one of --account or --accounts"); + if ((action === "get" || action === "infrastructure") && !account) throw new Error(`runtime ${action} requires --account `); + if ((action === "apply" || action === "delete") && account === null && accounts.length === 0) throw new Error(`runtime ${action} requires --account or --accounts`); + if (accounts.length > 0 && action !== "apply" && action !== "delete") throw new Error(`runtime ${action} does not accept --accounts`); + if (account !== null && (account.length > 256 || /[\r\n]/u.test(account))) throw new Error("--account has an unsupported format"); + if (group !== null && (group.length > 256 || /[\r\n]/u.test(group))) throw new Error("--group has an unsupported format"); + if (platform !== null && (!/^[A-Za-z0-9._-]+$/u.test(platform) || platform.length > 64)) throw new Error("--platform has an unsupported format"); + if (pageToken !== null && (!/^\d+$/u.test(pageToken) || pageToken.length > 20)) throw new Error("--page-token must be a numeric group cursor"); + if (group !== null && allGroups) throw new Error("use only one of --group or --all-groups"); + if ((allGroups || pageToken !== null) && action !== "errors") throw new Error(`runtime ${action} does not accept all-groups pagination options`); + if ((group !== null || platform !== null) && action !== "errors" && action !== "infrastructure") throw new Error(`runtime ${action} does not accept group scope options`); + if (pageToken !== null && !allGroups) throw new Error("--page-token requires --all-groups"); + if (account !== null && allGroups) throw new Error("--account cannot be combined with --all-groups; use --group first"); + if (action === "apply" && !template) throw new Error("runtime apply requires --template "); + if (action !== "apply" && template !== null) throw new Error(`runtime ${action} does not accept --template`); + if (action !== "errors" && action !== "infrastructure" && (since !== "24h" || tail !== 50_000)) throw new Error(`runtime ${action} does not accept --since or --tail`); + if (!/^\d+[mhd]$/u.test(since)) throw new Error("--since must use m, h, or d"); + if (["list", "get", "errors", "infrastructure"].includes(action) && confirm) throw new Error(`runtime ${action} does not accept --confirm`); + if ([full, raw, json].filter(Boolean).length > 1) throw new Error("use only one of --json, --full, or --raw"); + return { action, account, accounts, group, allGroups, platform, pageToken, template, kind, confirm, full, raw, json, since, tail, targetId }; +} + +function parseAccountSelectors(value: string): string[] { + const selectors = value.split(",").map((item) => item.trim()).filter(Boolean); + if (selectors.length === 0 || selectors.length > 100) throw new Error("--accounts requires 1 to 100 comma-separated selectors"); + if (selectors.some((item) => item.length > 256 || /[\r\n]/u.test(item))) throw new Error("--accounts has an unsupported selector"); + if (new Set(selectors).size !== selectors.length) throw new Error("--accounts contains duplicate selectors"); + return selectors; +} + +function parseRuntimeKind(value: string): "temp-unschedulable" { + if (value !== "temp-unschedulable") throw new Error("--kind must be temp-unschedulable"); + return value; +} + +function parseRuntimeTail(value: string): number { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 100 || parsed > 200_000) throw new Error("--tail must be an integer from 100 to 200000"); + return parsed; +} diff --git a/scripts/src/platform-infra-sub2api-codex/runtime.ts b/scripts/src/platform-infra-sub2api-codex/runtime.ts index 077d1661..88ed5dcf 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime.ts @@ -7,27 +7,8 @@ import { protectedManualAccountsForTarget } from "./public-exposure"; import { renderSub2ApiTempUnschedulableCredentials } from "./redaction"; import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote"; import { renderedCliResult, renderTable } from "./render"; -import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target"; - -type RuntimeAction = "list" | "get" | "errors" | "apply" | "delete"; - -interface RuntimeOptions { - action: RuntimeAction; - account: string | null; - group: string | null; - allGroups: boolean; - platform: string | null; - pageToken: string | null; - template: string | null; - kind: "temp-unschedulable"; - confirm: boolean; - full: boolean; - raw: boolean; - json: boolean; - since: string; - tail: number; - targetId: string; -} +import { parseRuntimeOptions, type RuntimeOptions } from "./runtime-options"; +import { codexPoolRuntimeTarget } from "./runtime-target"; export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { const options = parseRuntimeOptions(args); @@ -40,6 +21,7 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P const payload = { action: options.action, account: options.account, + accounts: options.accounts, group: options.group, allGroups: options.allGroups, platform: options.platform, @@ -90,6 +72,7 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P request: { operation: options.action, account: options.account, + accounts: options.accounts, group: options.group, allGroups: options.allGroups, platform: options.platform, @@ -97,8 +80,8 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P template: options.template, kind: options.kind, confirm: options.confirm, - since: options.action === "errors" ? options.since : undefined, - tail: options.action === "errors" ? options.tail : undefined, + since: options.action === "errors" || options.action === "infrastructure" ? options.since : undefined, + tail: options.action === "errors" || options.action === "infrastructure" ? options.tail : undefined, }, runtime: compactRuntimeErrors(parsed, options), remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }), @@ -159,6 +142,54 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; const groupFlag = options.group === null ? "" : ` --group ${options.group}`; const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; + if (options.json && options.account === null) { + return { + ok: runtime.ok, + operation: runtime.operation, + mutation: runtime.mutation, + group: runtime.group, + endpoint: runtime.endpoint, + errors: { + window: errors.window, + nativeOps: { + source: nativeOps.source, + overview: nativeOps.overview, + accountQuality: compactAccountQuality(nativeOps.accountQuality, false), + accountAvailability: { + status: availability.status, + totalAccounts: availabilityGroup.total_accounts, + availableCount: availabilityGroup.available_count, + rateLimitCount: availabilityGroup.rate_limit_count, + errorCount: availabilityGroup.error_count, + unavailableAccounts: Array.isArray(availability.unavailableAccounts) + ? availability.unavailableAccounts.map((item) => { + const account = runtimeRecord(item) ?? {}; + return { + accountId: account.accountId, + accountName: account.accountName, + status: account.status, + reason: account.error, + }; + }) + : [], + }, + concurrency: { + status: concurrency.status, + currentInUse: concurrencyGroup.current_in_use, + maxCapacity: concurrencyGroup.max_capacity, + loadPercentage: concurrencyGroup.load_percentage, + waitingInQueue: concurrencyGroup.waiting_in_queue, + }, + }, + }, + disclosure: { + account: `${baseCommand}${groupFlag}${platformFlag} --account `, + trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", + note: "This group overview keeps every account name and quality summary. Use account drilldown for request indexes and complete failure evidence.", + }, + valuesPrinted: false, + }; + } return { ok: runtime.ok, operation: runtime.operation, @@ -170,9 +201,10 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown nativeOps: { source: nativeOps.source, overview: nativeOps.overview, - responseHeaderTimeout: nativeOps.responseHeaderTimeout, - requestTiming: nativeOps.requestTiming, - customerErrors: nativeOps.customerErrors, + responseHeaderTimeout: options.account === null || !options.json ? nativeOps.responseHeaderTimeout : undefined, + requestTiming: options.account === null || !options.json ? compactRequestTiming(nativeOps.requestTiming) : undefined, + customerErrors: compactCustomerErrors(nativeOps.customerErrors, options.account !== null || !options.json), + accountQuality: compactAccountQuality(nativeOps.accountQuality, options.account !== null || !options.json), accountAvailability: { status: availability.status, totalAccounts: availabilityGroup.total_accounts, @@ -215,15 +247,11 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown failover: effectiveness.failover, forwardFailure: effectiveness.forwardFailure, selectFailedEvents: effectiveness.selectFailedEvents, - requestIdCoverage: effectiveness.requestIdCoverage, - gatewayOutcomeEvidence: effectiveness.gatewayOutcomeEvidence, requestIndexes, sourceStatus: { status: sourceStatus.status, - source: sourceStatus.source, fallbackUsed: sourceStatus.fallbackUsed, fallbackReason: sourceStatus.fallbackReason, - fallbackEventCount: sourceStatus.fallbackEventCount, }, }, }, @@ -237,6 +265,99 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown }; } +function compactRequestTiming(value: unknown): unknown { + const timing = runtimeRecord(value); + if (timing === null) return value; + return { + status: timing.status, + reason: timing.reason, + source: timing.source, + ttft: timing.ttft, + duration: timing.duration, + observedFloorSeconds: timing.observedFloorSeconds, + attribution: timing.attribution, + }; +} + +function compactCustomerErrors(value: unknown, detailed: boolean): unknown { + const profile = runtimeRecord(value); + if (profile === null) return value; + return { + status: profile.status, + reason: profile.reason, + source: profile.source, + total: profile.total, + modelBuckets: runtimeRecords(profile.modelBuckets).slice(0, 5), + accountBuckets: detailed ? runtimeRecords(profile.accountBuckets).slice(0, 5) : undefined, + statusBuckets: runtimeRecords(profile.statusBuckets).slice(0, 5), + streamBuckets: runtimeRecords(profile.streamBuckets).slice(0, 5), + }; +} + +function compactAccountQuality(value: unknown, detailed: boolean): unknown { + const quality = runtimeRecord(value); + if (quality === null) return value; + return { + source: quality.source, + scope: quality.scope, + window: quality.window, + scoreRule: (() => { + const rule = runtimeRecord(quality.scoreRule) ?? {}; + return detailed ? { + reliabilityWeight: rule.reliabilityWeight, + latencyWeight: rule.latencyWeight, + availabilityWeight: rule.availabilityWeight, + failureDefinition: rule.failureDefinition, + missingMetric: rule.missingMetric, + } : { + reliabilityWeight: rule.reliabilityWeight, + latencyWeight: rule.latencyWeight, + availabilityWeight: rule.availabilityWeight, + }; + })(), + accounts: runtimeRecords(quality.accounts).map((account) => detailed ? { + accountId: account.accountId, + accountName: account.accountName, + status: account.status, + currentlyAvailable: account.currentlyAvailable, + score: account.score, + grade: account.grade, + assessment: account.assessment, + confidence: account.confidence, + observedAttempts: account.observedAttempts, + successRequests: account.successRequests, + failureRequests: account.failureRequests, + failureRate: account.failureRate, + firstTokenSamples: account.firstTokenSamples, + firstTokenCoverage: account.firstTokenCoverage, + ttftP95Ms: account.ttftP95Ms, + customerErrorRequests: account.customerErrorRequests, + failoverRequests: account.failoverRequests, + failoverRecovered: account.failoverRecovered, + failoverFailed: account.failoverFailed, + sameAccountRetryEvents: account.sameAccountRetryEvents, + tempUnschedulableEvents: account.tempUnschedulableEvents, + forwardFailedRequests: account.forwardFailedRequests, + reasons: account.reasons, + } : { + accountId: account.accountId, + accountName: account.accountName, + currentlyAvailable: account.currentlyAvailable, + score: account.score, + grade: account.grade, + confidence: account.confidence, + observedAttempts: account.observedAttempts, + failureRate: account.failureRate, + ttftP95Ms: account.ttftP95Ms, + customerErrorRequests: account.customerErrorRequests, + failoverRequests: account.failoverRequests, + failoverRecovered: account.failoverRecovered, + failoverFailed: account.failoverFailed, + }), + attribution: detailed ? quality.attribution : undefined, + }; +} + function runtimeRecord(value: unknown): Record | null { return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : null; } @@ -269,6 +390,7 @@ function renderRuntimeResult(response: Record, options: Runtime const ok = response.ok === true; const lines: string[] = []; if (options.action === "errors") renderRuntimeErrors(lines, runtime, options); + else if (options.action === "infrastructure") renderRuntimeInfrastructure(lines, runtime); else if (options.action === "list") renderRuntimeAccountList(lines, runtime); else if (options.action === "get") renderRuntimeAccountDetail(lines, runtime); else renderRuntimeMutation(lines, runtime, options); @@ -327,6 +449,7 @@ function renderRuntimeErrors(lines: string[], runtime: Record, const ttft = runtimeRecord(timing.ttft) ?? {}; const duration = runtimeRecord(timing.duration) ?? {}; const customerErrors = runtimeRecord(nativeOps.customerErrors) ?? {}; + const accountQuality = runtimeRecord(nativeOps.accountQuality) ?? {}; lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`); lines.push(renderTable([ ["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"], @@ -358,6 +481,32 @@ function renderRuntimeErrors(lines: string[], runtime: Record, ["MODEL", "REQUESTS", "FIRST_TOKEN_ROWS", "COVERAGE", "AVG_TTFT_MS", "AVG_DURATION_MS", "AVG_TOKENS_S"], ...modelTiming.map((item) => [runtimeShort(item.model, 32), runtimeText(item.requestCount), runtimeText(item.requestsWithFirstToken), runtimeRate(item.firstTokenCoverage), runtimeText(item.avgFirstTokenMs), runtimeText(item.avgDurationMs), runtimeText(item.avgTokensPerSec)]), ])); + const qualityAccounts = runtimeRecords(accountQuality.accounts); + if (qualityAccounts.length > 0) { + lines.push("", "ACCOUNT QUALITY", renderTable([ + ["ACCOUNT", "ID", "SCORE", "GRADE", "QUALITY", "CONF", "ATTEMPTS", "SUCCESS", "FAIL", "FAIL_RATE", "TTFT_P95_MS", "TTFT_N", "AVAILABLE", "REASONS"], + ...qualityAccounts.map((item) => [ + runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.score), runtimeText(item.grade), runtimeText(item.assessment), runtimeText(item.confidence), + runtimeText(item.observedAttempts), runtimeText(item.successRequests), runtimeText(item.failureRequests), runtimeRate(item.failureRate), + runtimeText(item.ttftP95Ms), runtimeText(item.firstTokenSamples), runtimeText(item.currentlyAvailable), + runtimeShort(Array.isArray(item.reasons) ? item.reasons.join(",") : "", 58), + ]), + ])); + const controlRows = qualityAccounts.filter((item) => [ + item.failoverRequests, item.sameAccountRetryEvents, item.tempUnschedulableEvents, item.forwardFailedRequests, item.customerErrorRequests, + ].some((value) => typeof value === "number" && value > 0)); + if (controlRows.length > 0) lines.push("ACCOUNT FAILURE CONTROL", renderTable([ + ["ACCOUNT", "ID", "FAILOVER", "RECOVERED", "FAILED", "MISSING", "RETRY", "TEMP_UNSCHED", "FORWARD_FAIL", "CUSTOMER_ERR"], + ...controlRows.map((item) => [ + runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.failoverRequests), runtimeText(item.failoverRecovered), + runtimeText(item.failoverFailed), runtimeText(item.failoverOutcomeMissing), runtimeText(item.sameAccountRetryEvents), + runtimeText(item.tempUnschedulableEvents), runtimeText(item.forwardFailedRequests), runtimeText(item.customerErrorRequests), + ]), + ])); + const scoreRule = runtimeRecord(accountQuality.scoreRule) ?? {}; + lines.push(`SCORE RULE reliability=${runtimeText(scoreRule.reliabilityWeight)} latency=${runtimeText(scoreRule.latencyWeight)} availability=${runtimeText(scoreRule.availabilityWeight)} confidence=separate`); + if (typeof accountQuality.attribution === "string") lines.push(`NOTE: ${accountQuality.attribution}`); + } const errorModels = runtimeRecords(customerErrors.modelBuckets); const errorAccounts = runtimeRecords(customerErrors.accountBuckets); const errorDimensions = [ @@ -462,7 +611,100 @@ function renderRuntimeAccountDetail(lines: string[], runtime: Record): void { + const account = runtimeRecord(runtime.account) ?? {}; + const infra = runtimeRecord(runtime.infrastructure) ?? {}; + const proxy = runtimeRecord(infra.accountProxy) ?? {}; + const app = runtimeRecord(infra.appRuntime) ?? {}; + const route = runtimeRecord(infra.defaultRoute) ?? {}; + const dns = runtimeRecord(infra.dns) ?? {}; + const sockets = runtimeRecord(infra.proxySockets) ?? {}; + const listener = runtimeRecord(infra.proxyEndpointOwner) ?? {}; + const serviceLogs = runtimeRecord(infra.proxyServiceEvidence) ?? {}; + const logs = runtimeRecord(infra.recentNetworkEvidence) ?? {}; + lines.push(`SUB2API RUNTIME INFRASTRUCTURE ok=${runtime.ok === true ? "true" : "false"} since=${runtimeText(logs.window)}`); + if (runtime.ok !== true && typeof runtime.error === "string") lines.push(`ERROR: ${runtime.error}`); + lines.push(renderTable([ + ["ACCOUNT", "ID", "TYPE", "STATUS", "SCHED", "PROXY_ID", "PROXY_NAME", "PROTOCOL", "PROXY_ENDPOINT", "PROXY_STATUS"], + [ + runtimeText(account.accountName), runtimeText(account.accountId), runtimeText(account.type), runtimeText(account.status), + runtimeText(account.schedulable), runtimeText(proxy.id), runtimeText(proxy.name), runtimeText(proxy.protocol), + `${runtimeText(proxy.host)}:${runtimeText(proxy.port)}`, runtimeText(proxy.status), + ], + ])); + lines.push("", "APP RUNTIME", renderTable([ + ["MODE", "CONTAINER", "STATE", "RESTARTS", "OOM", "NETWORK_MODE", "STARTED_AT"], + [ + runtimeText(app.runtimeMode), runtimeText(app.name), runtimeText(app.state), runtimeText(app.restartCount), + runtimeText(app.oomKilled), runtimeText(app.networkMode), runtimeText(app.startedAt), + ], + ])); + const envRows = runtimeRecords(infra.proxyEnvironment); + if (envRows.length > 0) lines.push("PROXY ENVIRONMENT", renderTable([ + ["VARIABLE", "PRESENT", "SCHEME", "HOST", "PORT", "CREDENTIALS_REDACTED"], + ...envRows.map((item) => [runtimeText(item.name), runtimeText(item.present), runtimeText(item.scheme), runtimeText(item.host), runtimeText(item.port), runtimeText(item.credentialsRedacted)]), + ])); + lines.push("NETWORK", renderTable([ + ["DEFAULT_IFACE", "DEFAULT_GATEWAY", "DNS_NAMESERVERS", "DNS_SEARCH", "PROXY_RESOLVED_IPS", "SOCKETS", "ESTABLISHED"], + [ + runtimeText(route.interface), runtimeText(route.gateway), runtimeText(dns.nameservers), runtimeText(dns.search), + runtimeText(sockets.resolvedIps), runtimeText(sockets.total), runtimeText(sockets.established), + ], + ])); + const listeners = runtimeRecords(listener.listeners); + if (listeners.length > 0) lines.push("PROXY ENDPOINT OWNER", renderTable([ + ["LISTEN", "PROCESS", "PID", "UNIT", "ACTIVE", "RESTARTS", "TCP_PEERS", "UDP_PEERS"], + ...listeners.map((item) => [runtimeText(item.localEndpoint), runtimeText(item.processName), runtimeText(item.pid), runtimeText(item.systemdUnit), runtimeText(item.activeState), runtimeText(item.serviceRestarts), runtimeText(item.establishedPeers), runtimeText(item.udpPeers)]), + ])); + const serviceBuckets = runtimeRecords(serviceLogs.buckets); + if (serviceBuckets.length > 0) lines.push("PROXY SERVICE EVIDENCE", renderTable([ + ["CAUSE", "COUNT"], + ...serviceBuckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), + ])); + const serviceSamples = runtimeRecords(serviceLogs.samples); + if (serviceSamples.length > 0) lines.push("PROXY SERVICE INDEX", renderTable([ + ["AT", "CAUSE", "MESSAGE"], + ...serviceSamples.map((item) => [runtimeText(item.at), runtimeText(item.cause), runtimeShort(item.message, 100)]), + ])); + const interfaces = runtimeRecords(infra.interfaces); + if (interfaces.length > 0) lines.push("INTERFACES", renderTable([ + ["NAME", "RX_BYTES", "TX_BYTES", "RX_ERRORS", "TX_ERRORS", "RX_DROPPED", "TX_DROPPED"], + ...interfaces.map((item) => [runtimeText(item.name), runtimeText(item.rxBytes), runtimeText(item.txBytes), runtimeText(item.rxErrors), runtimeText(item.txErrors), runtimeText(item.rxDropped), runtimeText(item.txDropped)]), + ])); + const buckets = runtimeRecords(logs.buckets); + lines.push("", `RECENT NETWORK EVIDENCE matched=${runtimeText(logs.matchedEvents)} source=${runtimeText(logs.source)}`); + if (buckets.length > 0) lines.push(renderTable([ + ["CAUSE", "COUNT"], + ...buckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), + ])); + const samples = runtimeRecords(logs.samples); + if (samples.length > 0) lines.push("EVIDENCE INDEX", renderTable([ + ["AT", "REQUEST_ID", "CAUSE", "STATUS", "MESSAGE"], + ...samples.map((item) => [runtimeText(item.at), runtimeText(item.requestId), runtimeText(item.cause), runtimeText(item.statusCode), runtimeShort(item.message, 90)]), + ])); + if (typeof infra.attribution === "string") lines.push(`NOTE: ${infra.attribution}`); +} + function renderRuntimeMutation(lines: string[], runtime: Record, options: RuntimeOptions): void { + const plans = runtimeRecords(runtime.plans); + if (plans.length > 0) { + const summary = runtimeRecord(runtime.summary) ?? {}; + lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); + lines.push(renderTable([ + ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES", "RESULT"], + ...plans.map((plan) => { + const account = runtimeRecord(plan.account) ?? {}; + const before = runtimeRecord(plan.before) ?? {}; + const after = runtimeRecord(plan.after) ?? {}; + return [ + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), + runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), runtimeText(plan.result), + ]; + }), + ])); + lines.push(`SUMMARY selected=${runtimeText(summary.selected)} changed=${runtimeText(summary.changed)} succeeded=${runtimeText(summary.succeeded)} failed=${runtimeText(summary.failed)}`); + return; + } const plan = runtimeRecord(runtime.plan) ?? {}; const account = runtimeRecord(plan.account) ?? runtimeRecord(runtime.account) ?? {}; const before = runtimeRecord(plan.before) ?? {}; @@ -477,90 +719,6 @@ function renderRuntimeMutation(lines: string[], runtime: Record ])); } -export function parseRuntimeOptions(args: string[]): RuntimeOptions { - const [actionRaw = "list", ...rest] = args; - if (actionRaw !== "list" && actionRaw !== "get" && actionRaw !== "errors" && actionRaw !== "apply" && actionRaw !== "delete") { - throw new Error("runtime usage: list|get|errors|apply|delete [--account ] [--group |--all-groups] [--platform ] [--page-token ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--json|--full|--raw]"); - } - let account: string | null = null; - let group: string | null = null; - let allGroups = false; - let platform: string | null = null; - let pageToken: string | null = null; - let template: string | null = null; - let kind = "temp-unschedulable" as const; - let confirm = false; - let full = false; - let raw = false; - let json = false; - let since = "24h"; - let tail = 50_000; - let targetId = defaultCodexPoolRuntimeTargetId(); - for (let index = 0; index < rest.length; index += 1) { - const arg = rest[index]!; - const readValue = (name: string): string => { - const value = rest[index + 1]; - if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); - index += 1; - return value.trim(); - }; - if (arg === "--confirm") confirm = true; - else if (arg === "--full") full = true; - else if (arg === "--raw") raw = true; - else if (arg === "--json") json = true; - else if (arg === "-o" && readValue("-o") === "json") json = true; - else if (arg === "--account") account = readValue("--account"); - else if (arg.startsWith("--account=")) account = arg.slice("--account=".length).trim(); - else if (arg === "--group") group = readValue("--group"); - else if (arg.startsWith("--group=")) group = arg.slice("--group=".length).trim(); - else if (arg === "--all-groups") allGroups = true; - else if (arg === "--platform") platform = readValue("--platform"); - else if (arg.startsWith("--platform=")) platform = arg.slice("--platform=".length).trim(); - else if (arg === "--page-token") pageToken = readValue("--page-token"); - else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim(); - else if (arg === "--template") template = readValue("--template"); - else if (arg.startsWith("--template=")) template = arg.slice("--template=".length).trim(); - else if (arg === "--kind") kind = parseRuntimeKind(readValue("--kind")); - else if (arg.startsWith("--kind=")) kind = parseRuntimeKind(arg.slice("--kind=".length)); - else if (arg === "--target") targetId = readValue("--target"); - else if (arg.startsWith("--target=")) targetId = arg.slice("--target=".length).trim(); - else if (arg === "--since") since = readValue("--since"); - else if (arg.startsWith("--since=")) since = arg.slice("--since=".length).trim(); - else if (arg === "--tail") tail = parseRuntimeTail(readValue("--tail")); - else if (arg.startsWith("--tail=")) tail = parseRuntimeTail(arg.slice("--tail=".length)); - else throw new Error(`unsupported runtime option: ${arg}`); - } - if ((actionRaw === "get" || actionRaw === "apply" || actionRaw === "delete") && !account) { - throw new Error(`runtime ${actionRaw} requires --account `); - } - if (account !== null && (account.length > 256 || /[\r\n]/u.test(account))) throw new Error("--account has an unsupported format"); - if (group !== null && (group.length > 256 || /[\r\n]/u.test(group))) throw new Error("--group has an unsupported format"); - if (platform !== null && (!/^[A-Za-z0-9._-]+$/u.test(platform) || platform.length > 64)) throw new Error("--platform has an unsupported format"); - if (pageToken !== null && (!/^\d+$/u.test(pageToken) || pageToken.length > 20)) throw new Error("--page-token must be a numeric group cursor"); - if (group !== null && allGroups) throw new Error("use only one of --group or --all-groups"); - if ((group !== null || allGroups || platform !== null || pageToken !== null) && actionRaw !== "errors") throw new Error(`runtime ${actionRaw} does not accept group scope options`); - if (pageToken !== null && !allGroups) throw new Error("--page-token requires --all-groups"); - if (account !== null && allGroups) throw new Error("--account cannot be combined with --all-groups; use --group first"); - if (actionRaw === "apply" && !template) throw new Error("runtime apply requires --template "); - if (actionRaw !== "apply" && template !== null) throw new Error(`runtime ${actionRaw} does not accept --template`); - if (actionRaw !== "errors" && (since !== "24h" || tail !== 50_000)) throw new Error(`runtime ${actionRaw} does not accept --since or --tail`); - if (!/^\d+[mhd]$/u.test(since)) throw new Error("--since must use m, h, or d"); - if ((actionRaw === "list" || actionRaw === "get" || actionRaw === "errors") && confirm) throw new Error(`runtime ${actionRaw} does not accept --confirm`); - if ([full, raw, json].filter(Boolean).length > 1) throw new Error("use only one of --json, --full, or --raw"); - return { action: actionRaw, account, group, allGroups, platform, pageToken, template, kind, confirm, full, raw, json, since, tail, targetId }; -} - -function parseRuntimeKind(value: string): "temp-unschedulable" { - if (value !== "temp-unschedulable") throw new Error("--kind must be temp-unschedulable"); - return value; -} - -function parseRuntimeTail(value: string): number { - const parsed = Number(value); - if (!Number.isInteger(parsed) || parsed < 100 || parsed > 200_000) throw new Error("--tail must be an integer from 100 to 200000"); - return parsed; -} - function pyJson(value: unknown): string { return `json.loads(${JSON.stringify(JSON.stringify(value))})`; } @@ -571,10 +729,12 @@ set -u python3 - <<'PY' import json import re +import socket +import struct import subprocess import sys from datetime import datetime, timedelta, timezone -from urllib.parse import quote +from urllib.parse import quote, urlsplit RUNTIME_MODE = ${pyJson(target.runtimeMode)} NAMESPACE = ${pyJson(target.namespace)} @@ -790,6 +950,369 @@ def account_summary(detail, include_rules=False): result["tempUnschedulable"]["rules"] = policy["rules"] return result +def list_proxies(token): + return items_of(data_of(curl_api("GET", "/api/v1/admin/proxies?page=1&page_size=500", bearer=token), "list proxies")) + +def proxy_summary(token, proxy_id): + if proxy_id is None: + return {"configured": False, "route": "direct", "credentialsPrinted": False} + matches = [item for item in list_proxies(token) if isinstance(item, dict) and item.get("id") == proxy_id] + if len(matches) != 1: + return {"configured": True, "id": proxy_id, "status": "record-not-found", "credentialsPrinted": False} + item = matches[0] + return { + "configured": True, + "route": "account-proxy", + "id": item.get("id"), + "name": item.get("name"), + "protocol": item.get("protocol"), + "host": item.get("host"), + "port": item.get("port"), + "status": item.get("status"), + "fallbackMode": item.get("fallback_mode"), + "lastCheckAt": item.get("last_check_at"), + "lastError": text(item.get("last_error"), 240) if item.get("last_error") else None, + "credentialsPrinted": False, + } + +def proxy_environment_summary(): + rows = [] + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + value = app_text(["sh", "-c", f"printenv {name} 2>/dev/null || true"]) + parsed = urlsplit(value) if value else None + rows.append({ + "name": name, + "present": bool(value), + "scheme": parsed.scheme if parsed else None, + "host": parsed.hostname if parsed else None, + "port": parsed.port if parsed else None, + "credentialsRedacted": bool(parsed and (parsed.username or parsed.password)), + }) + no_proxy = app_text(["sh", "-c", "printenv NO_PROXY 2>/dev/null || printenv no_proxy 2>/dev/null || true"]) + return rows, { + "present": bool(no_proxy), + "entryCount": len([item for item in no_proxy.split(",") if item.strip()]), + "hyueapiPresent": any(item.strip() in ("hyueapi.com", ".hyueapi.com") for item in no_proxy.split(",")), + "valuesPrinted": False, + } + +def app_runtime_summary(): + if RUNTIME_MODE == "host-docker": + proc = docker(["inspect", APP_POD]) + if proc.returncode != 0: + return {"runtimeMode": RUNTIME_MODE, "name": APP_POD, "status": "unavailable", "reason": text(proc.stderr, 240)} + values = json.loads(proc.stdout.decode("utf-8")) + item = values[0] if values else {} + state = item.get("State") or {} + host_config = item.get("HostConfig") or {} + networks = ((item.get("NetworkSettings") or {}).get("Networks") or {}) + return { + "runtimeMode": RUNTIME_MODE, + "name": APP_POD, + "state": state.get("Status"), + "health": ((state.get("Health") or {}).get("Status")), + "restartCount": item.get("RestartCount"), + "oomKilled": state.get("OOMKilled"), + "startedAt": state.get("StartedAt"), + "networkMode": host_config.get("NetworkMode"), + "networks": [ + {"name": name, "ipAddress": value.get("IPAddress"), "gateway": value.get("Gateway")} + for name, value in sorted(networks.items()) if isinstance(value, dict) + ], + } + pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], "sub2api pod") + spec = pod.get("spec") or {} + status = pod.get("status") or {} + restarts = sum(item.get("restartCount") or 0 for item in status.get("containerStatuses") or [] if isinstance(item, dict)) + return { + "runtimeMode": RUNTIME_MODE, + "name": APP_POD, + "state": status.get("phase"), + "restartCount": restarts, + "oomKilled": None, + "startedAt": status.get("startTime"), + "networkMode": "host" if spec.get("hostNetwork") is True else "pod", + "nodeName": spec.get("nodeName"), + "podIp": status.get("podIP"), + "dnsPolicy": spec.get("dnsPolicy"), + } + +def dns_summary(): + content = app_text(["cat", "/etc/resolv.conf"]) + nameservers = [] + search = [] + options = [] + for line in content.splitlines(): + fields = line.strip().split() + if not fields or fields[0].startswith("#"): + continue + if fields[0] == "nameserver": nameservers.extend(fields[1:]) + elif fields[0] == "search": search.extend(fields[1:]) + elif fields[0] == "options": options.extend(fields[1:]) + return {"nameservers": nameservers, "search": search, "options": options} + +def default_route_summary(): + content = app_text(["cat", "/proc/net/route"]) + for line in content.splitlines()[1:]: + fields = line.split() + if len(fields) < 4 or fields[1] != "00000000": + continue + try: + gateway = socket.inet_ntoa(struct.pack("= 5 and peer_fields[0] == "ESTAB": + peers.append(peer_fields[4]) + udp_proc = run(["ss", "-unpH"]) + if udp_proc.returncode == 0: + for udp_line in udp_proc.stdout.decode("utf-8", errors="replace").splitlines(): + if f"pid={pid}," not in udp_line: + continue + udp_fields = udp_line.split() + if len(udp_fields) >= 5 and udp_fields[4] not in ("*:*", "0.0.0.0:*"): + udp_peers.append(udp_fields[4]) + cgroup_proc = run(["cat", f"/proc/{pid}/cgroup"]) + cgroup_text = cgroup_proc.stdout.decode("utf-8", errors="replace") if cgroup_proc.returncode == 0 else "" + unit_match = re.search(r'/([^/]+\\.service)(?:/|$)', cgroup_text) + systemd_unit = unit_match.group(1) if unit_match else None + if systemd_unit: + show_proc = run(["systemctl", "show", systemd_unit, "--property=ActiveState", "--property=SubState", "--property=NRestarts"]) + if show_proc.returncode == 0: + service_values = dict(line.split("=", 1) for line in show_proc.stdout.decode("utf-8", errors="replace").splitlines() if "=" in line) + active_state = "/".join(value for value in (service_values.get("ActiveState"), service_values.get("SubState")) if value) + service_restarts = service_values.get("NRestarts") + rows.append({ + "localEndpoint": local_endpoint, + "processName": process_name, + "pid": pid, + "executable": executable, + "establishedPeers": sorted(set(peers))[:12], + "udpPeers": sorted(set(udp_peers))[:12], + "systemdUnit": systemd_unit, + "activeState": active_state, + "serviceRestarts": service_restarts, + }) + return {"status": "available", "listeners": rows, "credentialsPrinted": False} + +def proxy_service_evidence(owner): + listeners = owner.get("listeners") if isinstance(owner, dict) else [] + units = sorted(set(item.get("systemdUnit") for item in listeners if isinstance(item, dict) and item.get("systemdUnit"))) + if not units: + return {"status": "not-applicable", "buckets": [], "samples": []} + buckets = {} + samples = [] + for unit in units: + proc = run(["journalctl", "--unit", unit, "--since", since_start_time(PAYLOAD.get("since")), "--no-pager", "--output", "json", "--lines", "2000"]) + if proc.returncode != 0: + continue + for line in proc.stdout.decode("utf-8", errors="replace").splitlines(): + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + message = str(item.get("MESSAGE") or "") + lowered = message.lower() + priority = int(item.get("PRIORITY")) if str(item.get("PRIORITY") or "").isdigit() else 7 + cause = None + if "reconnect" in lowered or "retry" in lowered: cause = "reconnect" + elif "disconnect" in lowered or "closed" in lowered: cause = "disconnected" + elif "timeout" in lowered or "deadline" in lowered: cause = "timeout" + elif "error" in lowered or "failed" in lowered or priority <= 3: cause = "service-error" + if cause is None: + continue + buckets[cause] = buckets.get(cause, 0) + 1 + if len(samples) < (30 if PAYLOAD.get("full") is True else 12): + samples.append({"at": item.get("__REALTIME_TIMESTAMP"), "cause": cause, "message": text(message, 400)}) + return { + "status": "available", + "units": units, + "buckets": [{"cause": key, "count": value} for key, value in sorted(buckets.items(), key=lambda pair: (-pair[1], pair[0]))], + "samples": samples, + } + +def classify_network_message(message, status_code): + lowered = message.lower() + checks = ( + ("dns-failure", ("no such host", "name resolution", "temporary failure in name resolution")), + ("proxy-connect-failure", ("proxyconnect", "connect tunnel", "http connect")), + ("connection-reset", ("connection reset", "reset by peer")), + ("unexpected-eof", ("unexpected eof", " eof")), + ("broken-pipe", ("broken pipe",)), + ("connection-refused", ("connection refused",)), + ("network-unreachable", ("network is unreachable", "no route to host")), + ("transport-timeout", ("i/o timeout", "tls handshake timeout", "context deadline exceeded")), + ("context-canceled", ("context canceled", "context cancelled")), + ("stream-incomplete", ("missing terminal event", "stream usage incomplete", "response failed")), + ) + for cause, markers in checks: + if any(marker in lowered for marker in markers): + return cause + return "upstream-503" if status_code == 503 or " 503" in lowered else None + +def recent_network_evidence(account_id): + lines = runtime_log_lines() + buckets = {} + samples = [] + matched = 0 + sample_limit = 30 if PAYLOAD.get("full") is True else 12 + for raw in lines: + candidate = raw[raw.find("{"):] if "{" in raw else raw + try: + item = json.loads(candidate) + except json.JSONDecodeError: + item = {} + extra = item.get("extra") if isinstance(item.get("extra"), dict) else {} + observed_account = item.get("account_id") if item.get("account_id") is not None else extra.get("account_id") + if str(observed_account) != str(account_id): + continue + status_code = item.get("status_code") if isinstance(item.get("status_code"), int) else extra.get("status_code") + if not isinstance(status_code, int): + upstream_status = extra.get("upstream_status") + status_code = upstream_status if isinstance(upstream_status, int) else None + message_parts = [item.get("message"), item.get("error"), extra.get("error"), extra.get("message")] + message = " | ".join(str(value) for value in message_parts if value not in (None, "")) + cause = classify_network_message(message, status_code) + if cause is None: + continue + matched += 1 + buckets[cause] = buckets.get(cause, 0) + 1 + if len(samples) < sample_limit: + samples.append({ + "at": item.get("time") or item.get("created_at") or extra.get("_at"), + "requestId": item.get("request_id") or extra.get("request_id"), + "cause": cause, + "statusCode": status_code, + "message": text(message, 360), + }) + return { + "source": "sub2api-target-runtime-logs", + "window": PAYLOAD.get("since"), + "scannedLines": len(lines), + "matchedEvents": matched, + "buckets": [{"cause": key, "count": value} for key, value in sorted(buckets.items(), key=lambda pair: (-pair[1], pair[0]))], + "samples": samples, + } + +def infrastructure_snapshot(token, detail): + proxy = proxy_summary(token, detail.get("proxy_id")) + proxy_env, no_proxy = proxy_environment_summary() + owner = proxy_endpoint_owner(proxy) + return { + "source": "sub2api-admin-api-and-target-runtime", + "accountProxy": proxy, + "appRuntime": app_runtime_summary(), + "proxyEnvironment": proxy_env, + "noProxy": no_proxy, + "dns": dns_summary(), + "defaultRoute": default_route_summary(), + "interfaces": interface_summaries(), + "proxySockets": proxy_socket_summary(proxy), + "proxyEndpointOwner": owner, + "proxyServiceEvidence": proxy_service_evidence(owner), + "recentNetworkEvidence": recent_network_evidence(detail.get("id")), + "attribution": "The account proxy record controls this account's transport. Container proxy environment is reported separately. Interface counters and sockets are query-time snapshots; no upstream account test or application request is sent.", + "valuesPrinted": False, + } + def policy_summary(policy, include_rules=False): keys = ("enabled", "ruleCount", "statusCodes", "rules") if include_rules else ("enabled", "ruleCount", "statusCodes") return {key: policy[key] for key in keys} @@ -1305,9 +1828,27 @@ def native_customer_error_profile(token, group_id, platform): counts[label] = counts.get(label, 0) + 1 return [{"value": key, "count": count} for key, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20 if PAYLOAD.get("full") is True else 8]] samples = [] + account_metrics = {} seen = set() for row in rows: - if not isinstance(row, dict) or not row.get("request_id"): + if not isinstance(row, dict): + continue + account_id = row.get("account_id") + account_key = str(account_id) if account_id is not None else "unknown" + if account_key not in account_metrics: + account_metrics[account_key] = { + "accountId": account_id, "customerErrorEvents": 0, "customerErrorRequestIds": set(), "statusCodes": {}, + } + metric = account_metrics[account_key] + metric["customerErrorEvents"] += 1 + request_id = row.get("request_id") + if isinstance(request_id, str) and request_id: + metric["customerErrorRequestIds"].add(request_id) + status_code = row.get("status_code") + if isinstance(status_code, int): + status_key = str(status_code) + metric["statusCodes"][status_key] = metric["statusCodes"].get(status_key, 0) + 1 + if not request_id: continue key = (row.get("requested_model") or row.get("model"), row.get("status_code"), row.get("account_id"), row.get("stream"), row.get("request_path") or row.get("inbound_endpoint")) if key in seen: @@ -1330,9 +1871,190 @@ def native_customer_error_profile(token, group_id, platform): "phaseBuckets": buckets(lambda row: row.get("phase")), "endpointBuckets": buckets(lambda row: row.get("inbound_endpoint") or row.get("request_path")), "pathBuckets": buckets(lambda row: row.get("request_path") or row.get("inbound_endpoint")), + "accountMetrics": [ + { + "accountId": item["accountId"], "customerErrorEvents": item["customerErrorEvents"], + "customerErrorRequests": len(item["customerErrorRequestIds"]), + "_requestIds": sorted(item["customerErrorRequestIds"]), + "statusBuckets": [ + {"statusCode": int(code), "count": count} + for code, count in sorted(item["statusCodes"].items(), key=lambda pair: int(pair[0])) + ], + } + for item in sorted(account_metrics.values(), key=lambda item: int(item.get("accountId") or 0)) + ], "samples": samples, } +def parse_native_time(value): + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc) + except ValueError: + return None + +def metric_percentile(values, fraction): + ordered = sorted(value for value in values if isinstance(value, (int, float)) and value >= 0) + if not ordered: + return None + position = (len(ordered) - 1) * fraction + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + return round(ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)) + +def account_usage_metrics(token, accounts, group_id): + start_at = parse_native_time(since_start_time(PAYLOAD.get("since"))) + end_at = datetime.now(timezone.utc) + result = {} + for account in accounts: + account_id = account.get("id") + key = str(account_id) + rows = [] + page = 1 + status = "available" + reason = None + try: + while True: + path = ( + f"/api/v1/admin/usage?page={page}&page_size=100&group_id={group_id}&account_id={account_id}" + f"&start_date={start_at.date().isoformat()}&end_date={end_at.date().isoformat()}&timezone=UTC" + "&sort_by=created_at&sort_order=desc" + ) + data = data_of(curl_api("GET", path, bearer=token), "list account usage") + page_rows = items_of(data) + reached_start = False + for row in page_rows: + created_at = parse_native_time(row.get("created_at") if isinstance(row, dict) else None) + if created_at is None or created_at < start_at or created_at > end_at: + if created_at is not None and created_at < start_at: + reached_start = True + continue + rows.append(row) + if not page_rows or len(page_rows) < 100 or reached_start: + break + page += 1 + except Exception as exc: + status = "unavailable" + reason = compact_error(str(exc)) + stream_rows = [row for row in rows if isinstance(row, dict) and row.get("stream") is True] + ttft_values = [row.get("first_token_ms") for row in stream_rows if isinstance(row.get("first_token_ms"), int)] + duration_values = [row.get("duration_ms") for row in rows if isinstance(row.get("duration_ms"), int)] + models = {} + for row in rows: + model = str(row.get("model") or "unknown") + models[model] = models.get(model, 0) + 1 + result[key] = { + "status": status, "reason": reason, "source": "sub2api-native-admin-usage", + "pagesRead": page, "successRequests": len(rows), "streamSuccessRequests": len(stream_rows), + "firstTokenSamples": len(ttft_values), + "firstTokenCoverage": round(len(ttft_values) / len(stream_rows), 6) if stream_rows else None, + "ttftP50Ms": metric_percentile(ttft_values, 0.50), "ttftP95Ms": metric_percentile(ttft_values, 0.95), + "ttftP99Ms": metric_percentile(ttft_values, 0.99), "ttftMaxMs": max(ttft_values) if ttft_values else None, + "durationP95Ms": metric_percentile(duration_values, 0.95), + "modelBuckets": [{"model": model, "count": count} for model, count in sorted(models.items(), key=lambda item: (-item[1], item[0]))[:8]], + } + return result + +def reliability_score_points(failure_rate): + if not isinstance(failure_rate, (int, float)): + return None + return round(60 * (1 - min(max(failure_rate, 0), 0.20) / 0.20), 2) + +def latency_score_points(ttft_p95_ms): + if not isinstance(ttft_p95_ms, (int, float)): + return None + return round(25 * (1 - min(max(ttft_p95_ms - 10000, 0), 170000) / 170000), 2) + +def account_quality_scores(token, accounts, group_id, native_ops, effectiveness): + usage_by_account = account_usage_metrics(token, accounts, group_id) + customer_profile = native_ops.get("customerErrors") if isinstance(native_ops.get("customerErrors"), dict) else {} + customer_by_account = {str(item.get("accountId")): item for item in customer_profile.get("accountMetrics") or [] if isinstance(item, dict)} + policy_by_account = {str(item.get("accountId")): item for item in effectiveness.get("accountPolicyMetrics") or [] if isinstance(item, dict)} + availability = ((native_ops.get("accountAvailability") or {}).get("summary") or {}) + unavailable_ids = set(str(item.get("accountId")) for item in availability.get("unavailableAccounts") or [] if isinstance(item, dict)) + rows = [] + for account in accounts: + account_id = account.get("id") + key = str(account_id) + usage = usage_by_account.get(key) or {} + customer = customer_by_account.get(key) or {} + policy = policy_by_account.get(key) or {} + policy_failure_ids = set(policy.pop("_failureRequestIds", []) or []) + customer_failure_ids = set(customer.pop("_requestIds", []) or []) + failure_requests = len(policy_failure_ids | customer_failure_ids) + success_requests = usage.get("successRequests") if isinstance(usage.get("successRequests"), int) else 0 + observed_attempts = success_requests + failure_requests + failure_rate = round(failure_requests / observed_attempts, 6) if observed_attempts > 0 else None + reliability_points = reliability_score_points(failure_rate) + ttft_samples = usage.get("firstTokenSamples") if isinstance(usage.get("firstTokenSamples"), int) else 0 + latency_points = latency_score_points(usage.get("ttftP95Ms")) if ttft_samples >= 5 else None + current_available = key not in unavailable_ids and account.get("status") == "active" and account.get("schedulable") is not False + availability_points = 15 if current_available else 8 if account.get("status") == "active" else 0 + earned = (reliability_points or 0) + (latency_points or 0) + availability_points + available_weight = (60 if reliability_points is not None else 0) + (25 if latency_points is not None else 0) + 15 + score = round(earned / available_weight * 100, 1) if observed_attempts > 0 and available_weight > 0 else None + score_comparable = observed_attempts >= 10 and ttft_samples >= 5 + grade = "insufficient" + if isinstance(score, (int, float)) and (score_comparable or (score < 60 and observed_attempts >= 10)): + grade = "A" if score >= 90 else "B" if score >= 80 else "C" if score >= 70 else "D" if score >= 60 else "E" + assessment = "preferred" if grade == "A" else "healthy" if grade == "B" else "watch" if grade == "C" else "degraded" if grade == "D" else "poor" if grade == "E" else "insufficient-evidence" + confidence = "high" if observed_attempts >= 50 and ttft_samples >= 20 else "medium" if observed_attempts >= 10 and ttft_samples >= 5 else "low" + reasons = [] + if isinstance(failure_rate, (int, float)) and failure_rate >= 0.10: + reasons.append("failure-rate>=10%") + elif isinstance(failure_rate, (int, float)) and failure_rate >= 0.03: + reasons.append("failure-rate>=3%") + if isinstance(usage.get("ttftP95Ms"), int) and usage.get("ttftP95Ms") >= 120000: + reasons.append("ttft-p95>=120s") + if policy.get("failoverRequests", 0) > 0: + reasons.append("upstream-failover-triggered") + if policy.get("sameAccountRetryEvents", 0) > 0: + reasons.append("same-account-retry-observed") + if not current_available: + reasons.append("currently-unavailable") + if ttft_samples < 5: + reasons.append("ttft-evidence-insufficient") + if observed_attempts < 10: + reasons.append("request-evidence-insufficient") + rows.append({ + "accountId": account_id, "accountName": account.get("name"), "status": account.get("status"), + "schedulable": account.get("schedulable"), "currentlyAvailable": current_available, + "score": score, "grade": grade, "assessment": assessment, "confidence": confidence, + "scoreComparable": score_comparable, + "observedAttempts": observed_attempts, "successRequests": success_requests, + "failureRequests": failure_requests, "failureRate": failure_rate, + "streamSuccessRequests": usage.get("streamSuccessRequests"), "firstTokenSamples": ttft_samples, + "firstTokenCoverage": usage.get("firstTokenCoverage"), "ttftP50Ms": usage.get("ttftP50Ms"), + "ttftP95Ms": usage.get("ttftP95Ms"), "ttftP99Ms": usage.get("ttftP99Ms"), "ttftMaxMs": usage.get("ttftMaxMs"), + "durationP95Ms": usage.get("durationP95Ms"), "modelBuckets": usage.get("modelBuckets"), + "customerErrorRequests": customer.get("customerErrorRequests", 0), + "failoverRequests": policy.get("failoverRequests", 0), "failoverRecovered": policy.get("failoverRecovered", 0), + "failoverFailed": policy.get("failoverFailed", 0), "failoverOutcomeMissing": policy.get("failoverOutcomeMissing", 0), + "sameAccountRetryEvents": policy.get("sameAccountRetryEvents", 0), + "tempUnschedulableEvents": policy.get("tempUnschedulableEvents", 0), + "forwardFailedRequests": policy.get("forwardFailedRequests", 0), + "upstreamStatusBuckets": policy.get("upstreamStatusBuckets", []), + "scoreComponents": {"reliability": reliability_points, "latency": latency_points, "availability": availability_points, "availableWeight": available_weight}, + "reasons": reasons, "usageStatus": usage.get("status"), "usageReason": usage.get("reason"), + }) + rows.sort(key=lambda item: (item.get("score") is None, -(item.get("score") or -1), int(item.get("accountId") or 0))) + return { + "source": "sub2api-native-admin-usage-request-errors-and-system-logs", + "scope": "group-account-window", "window": PAYLOAD.get("since"), + "scoreRule": { + "reliabilityWeight": 60, "latencyWeight": 25, "availabilityWeight": 15, + "failureDefinition": "unique request ids with account failover, forward_failed, or customer-visible error", + "reliability": "linear from 60 points at 0% observed failures to 0 points at 20%", + "latency": "requires at least 5 first-token samples; linear from 25 points at <=10s TTFT P95 to 0 points at >=180s", + "availability": "15 points when active, schedulable, and currently available; 8 when active but unavailable; otherwise 0", + "missingMetric": "missing latency is excluded from the denominator and lowers confidence; it is not treated as zero latency", + }, + "accounts": rows, + "attribution": "Success denominators and first-token latency come from native admin usage records. Failures are deduplicated by request id across native request-errors and indexed failover/forward events. Scores are analytical and never mutate scheduling.", + } + def all_group_error_overview(token, groups): page_size = 20 cursor = int(PAYLOAD.get("pageToken") or 0) @@ -1440,15 +2162,33 @@ def native_system_log_lines(token, platform=None): platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" markers = ( "account_temp_unschedulable", "openai.upstream_failover_switching", - "openai.account_select_failed", "openai.forward_failed", "http request completed", + "openai.account_select_failed", "openai.forward_failed", + "openai.pool_mode_same_account_retry", "gateway.failover_same_account_retry", + "http request completed", ) rows_by_id = {} queries = [] for marker in markers: - path = f"/api/v1/admin/ops/system-logs?page=1&page_size=200&start_time={start_time}&q={quote(marker, safe='')}{platform_query}" - response = curl_api("GET", path, bearer=token) + page = 1 + returned = 0 + total = None try: - data = data_of(response, "list ops system logs") + while True: + path = f"/api/v1/admin/ops/system-logs?page={page}&page_size=200&start_time={start_time}&q={quote(marker, safe='')}{platform_query}" + data = data_of(curl_api("GET", path, bearer=token), "list ops system logs") + rows = items_of(data) + returned += len(rows) + if total is None and isinstance(data, dict) and isinstance(data.get("total"), int): + total = data.get("total") + for row in rows: + if not isinstance(row, dict): + continue + row_id = row.get("id") + key = str(row_id) if row_id is not None else json.dumps(row, sort_keys=True, default=str) + rows_by_id[key] = row + if not rows or len(rows) < 200 or (isinstance(total, int) and returned >= total): + break + page += 1 except Exception as exc: return None, { "status": "unavailable", @@ -1456,15 +2196,7 @@ def native_system_log_lines(token, platform=None): "reason": compact_error(str(exc)), "fallback": "runtime-container-logs", } - rows = items_of(data) - total = data.get("total", len(rows)) if isinstance(data, dict) else len(rows) - queries.append({"marker": marker, "returned": len(rows), "total": total, "truncated": total > len(rows)}) - for row in rows: - if not isinstance(row, dict): - continue - row_id = row.get("id") - key = str(row_id) if row_id is not None else json.dumps(row, sort_keys=True, default=str) - rows_by_id[key] = row + queries.append({"marker": marker, "returned": returned, "total": total if total is not None else returned, "pagesRead": page, "truncated": False}) lines = [] for row in sorted(rows_by_id.values(), key=lambda item: str(item.get("created_at") or "")): extra = dict(row.get("extra") or {}) if isinstance(row.get("extra"), dict) else {} @@ -1487,12 +2219,23 @@ def native_system_log_lines(token, platform=None): def policy_effectiveness(lines, accounts, source, group_id=None): account_names = {str(item.get("id")): item.get("name") for item in accounts} account_ids = set(account_names.keys()) + account_policy = { + key: { + "accountId": item.get("id"), "accountName": item.get("name"), + "tempUnschedulableEvents": 0, "failoverEvents": 0, "failoverRequestIds": set(), + "failoverRecoveredRequestIds": set(), "failoverFailedRequestIds": set(), "failoverOutcomeMissingRequestIds": set(), + "forwardFailedEvents": 0, "forwardFailedRequestIds": set(), "sameAccountRetryEvents": 0, + "upstreamStatusCodes": {}, + } + for key, item in ((str(item.get("id")), item) for item in accounts) + } requests = {} counts = { "tempUnschedulableEvents": 0, "failoverEvents": 0, "selectFailedEvents": 0, "forwardFailedEvents": 0, + "sameAccountRetryEvents": 0, "gatewayCompletedEvents": 0, "gatewayFinalErrorEvents": 0, } @@ -1501,6 +2244,7 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "failover": 0, "selectFailed": 0, "forwardFailed": 0, + "sameAccountRetry": 0, "gatewayCompleted": 0, } temp_accounts = {} @@ -1520,6 +2264,7 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "failoverCount": 0, "selectFailedCount": 0, "forwardFailedCount": 0, + "sameAccountRetryCount": 0, "finalStatus": None, } return requests[request_id] @@ -1527,7 +2272,9 @@ def policy_effectiveness(lines, accounts, source, group_id=None): for line in lines: if not any(marker in line for marker in ( "account_temp_unschedulable", "openai.upstream_failover_switching", - "openai.account_select_failed", "openai.forward_failed", "http request completed", + "openai.account_select_failed", "openai.forward_failed", + "openai.pool_mode_same_account_retry", "gateway.failover_same_account_retry", + "http request completed", )): continue json_start = line.find("{") @@ -1557,6 +2304,8 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "accountId": account_id, "accountName": account_names.get(str(account_id)), } + raw_account_stat = account_policy.get(str(account_id)) + account_stat = raw_account_stat if group_id is None or event_group_id is not None else None if "account_temp_unschedulable" in line: event_line_count += 1 counts["tempUnschedulableEvents"] += 1 @@ -1572,6 +2321,8 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "count": 0, } temp_accounts[key]["count"] += 1 + if raw_account_stat is not None: + raw_account_stat["tempUnschedulableEvents"] += 1 if len(temp_samples) < 20: temp_samples.append({ **base, @@ -1588,6 +2339,14 @@ def policy_effectiveness(lines, accounts, source, group_id=None): request["failoverCount"] += 1 else: missing_request_ids["failover"] += 1 + if account_stat is not None: + account_stat["failoverEvents"] += 1 + if isinstance(request_id, str) and request_id: + account_stat["failoverRequestIds"].add(request_id) + status_code = item.get("upstream_status") + if isinstance(status_code, int): + key = str(status_code) + account_stat["upstreamStatusCodes"][key] = account_stat["upstreamStatusCodes"].get(key, 0) + 1 if len(failover_samples) < 20: failover_samples.append({ **base, @@ -1614,6 +2373,19 @@ def policy_effectiveness(lines, accounts, source, group_id=None): request["forwardFailedCount"] += 1 else: missing_request_ids["forwardFailed"] += 1 + if account_stat is not None: + account_stat["forwardFailedEvents"] += 1 + if isinstance(request_id, str) and request_id: + account_stat["forwardFailedRequestIds"].add(request_id) + elif "openai.pool_mode_same_account_retry" in line or "gateway.failover_same_account_retry" in line: + event_line_count += 1 + counts["sameAccountRetryEvents"] += 1 + if request is not None: + request["sameAccountRetryCount"] += 1 + else: + missing_request_ids["sameAccountRetry"] += 1 + if raw_account_stat is not None: + raw_account_stat["sameAccountRetryEvents"] += 1 elif "http request completed" in line and gateway_request_path(item.get("path")): event_line_count += 1 counts["gatewayCompletedEvents"] += 1 @@ -1645,6 +2417,37 @@ def policy_effectiveness(lines, accounts, source, group_id=None): item for item in requests.values() if item["forwardFailedCount"] > 0 and not isinstance(item.get("finalStatus"), int) ] + for request in correlated: + request_id = request.get("requestId") + if not isinstance(request_id, str): + continue + for current in account_policy.values(): + if request_id not in current["failoverRequestIds"]: + continue + final_status = request.get("finalStatus") + if isinstance(final_status, int) and final_status < 400: + current["failoverRecoveredRequestIds"].add(request_id) + elif isinstance(final_status, int): + current["failoverFailedRequestIds"].add(request_id) + else: + current["failoverOutcomeMissingRequestIds"].add(request_id) + account_policy_rows = [] + for current in account_policy.values(): + account_policy_rows.append({ + "accountId": current["accountId"], "accountName": current["accountName"], + "tempUnschedulableEvents": current["tempUnschedulableEvents"], + "failoverEvents": current["failoverEvents"], "failoverRequests": len(current["failoverRequestIds"]), + "failoverRecovered": len(current["failoverRecoveredRequestIds"]), + "failoverFailed": len(current["failoverFailedRequestIds"]), + "failoverOutcomeMissing": len(current["failoverOutcomeMissingRequestIds"]), + "forwardFailedEvents": current["forwardFailedEvents"], "forwardFailedRequests": len(current["forwardFailedRequestIds"]), + "sameAccountRetryEvents": current["sameAccountRetryEvents"], + "_failureRequestIds": sorted(current["failoverRequestIds"] | current["forwardFailedRequestIds"]), + "upstreamStatusBuckets": [ + {"statusCode": int(code), "count": count} + for code, count in sorted(current["upstreamStatusCodes"].items(), key=lambda pair: int(pair[0])) + ], + }) sample_limit = 20 if PAYLOAD.get("full") is True else (2 if PAYLOAD.get("account") is not None else 5) completed = counts["gatewayCompletedEvents"] final_errors = counts["gatewayFinalErrorEvents"] @@ -1655,6 +2458,8 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "eventLineCount": event_line_count, **counts, "tempUnschedulableAccounts": sorted(temp_accounts.values(), key=lambda item: (-item["count"], str(item.get("accountName") or ""))), + "accountPolicyMetrics": sorted(account_policy_rows, key=lambda item: int(item.get("accountId") or 0)), + "accountPolicyAttribution": "Failover and forward-failure request ids enter group scoring only when system-log events carry group_id. Account temp-unschedulable and same-account retry counts are account-wide operational signals and are not added to the failure-rate numerator.", "failoverRequestCount": len(correlated), "failoverSucceededRequestCount": len(succeeded), "failoverFailedRequestCount": len(failed), @@ -1746,12 +2551,13 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): system_log_source["fallbackUsed"] = True system_log_source["fallbackReason"] = "native system-log index returned no policy events while the native overview reported upstream errors" system_log_source["fallbackEventCount"] = len(runtime_event_lines) - effectiveness = policy_effectiveness(lines, accounts, system_log_source.get("source"), group_id) + effectiveness = policy_effectiveness(lines, selected if selector is not None else accounts, system_log_source.get("source"), group_id) effectiveness["sourceStatus"] = system_log_source overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) native_ops["responseHeaderTimeout"] = runtime_response_header_timeout() native_ops["requestTiming"] = native_request_timing(token, group_id, platform, overview_summary) native_ops["customerErrors"] = native_customer_error_profile(token, group_id, platform) + native_ops["accountQuality"] = account_quality_scores(token, selected, group_id, native_ops, effectiveness) request_total = overview_summary.get("request_count_total") error_total = overview_summary.get("error_count_total") if isinstance(request_total, int) and isinstance(error_total, int): @@ -1888,6 +2694,7 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): "responseHeaderTimeout": native_ops.get("responseHeaderTimeout"), "requestTiming": native_ops.get("requestTiming"), "customerErrors": native_ops.get("customerErrors"), + "accountQuality": native_ops.get("accountQuality"), "accountAvailability": { "status": (native_ops.get("accountAvailability") or {}).get("status"), "group": availability.get("group"), @@ -1981,6 +2788,37 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): "message": "observedStatusCodes comes only from account_upstream_error; forward failures are supplemental message evidence", } +def runtime_mutation_plan(detail, action): + if detail.get("type") != "apikey": + raise RuntimeError(f"runtime temp-unschedulable policy requires an apikey account: {detail.get('name')} ({detail.get('id')})") + credentials = dict(detail.get("credentials") or {}) + before = normalize_policy(credentials) + if action == "apply": + template = PAYLOAD.get("selectedTemplate") + if not isinstance(template, dict): + raise RuntimeError("selected runtime template missing") + desired = dict(credentials) + desired.update(template.get("credentials") or {}) + after = normalize_policy(desired) + exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials + change = "noop" if before == after else ("update" if exists else "create") + else: + desired = dict(credentials) + desired.pop("temp_unschedulable_enabled", None) + desired.pop("temp_unschedulable_rules", None) + after = normalize_policy(desired) + change = "delete" if before != after or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop" + include_rules = PAYLOAD.get("full") is True + plan = { + "change": change, + "account": account_summary(detail, include_rules), + "before": policy_summary(before, include_rules), + "after": policy_summary(after, include_rules), + "template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None, + "credentialsPrinted": False, + } + return plan, desired + def runtime_result(): token = login() action = PAYLOAD["action"] @@ -1996,8 +2834,8 @@ def runtime_result(): "valuesPrinted": False, } group_selector = PAYLOAD.get("group") - group_platform = PAYLOAD.get("platform") if action == "errors" else "openai" - if action == "errors" and group_selector is None: + group_platform = PAYLOAD.get("platform") if action in ("errors", "infrastructure") else "openai" + if action in ("errors", "infrastructure") and group_selector is None: group_platform = "openai" groups = list_groups(token, group_platform) selector = str(group_selector if group_selector is not None else PAYLOAD["groupName"]) @@ -2021,43 +2859,49 @@ def runtime_result(): if action == "errors": errors = observed_runtime_errors(token, accounts, group["id"], effective_platform) return {**base, "ok": True, "operation": "errors", "mutation": False, "errors": errors} - detail = find_account(token, accounts, str(PAYLOAD["account"])) + selectors = PAYLOAD.get("accounts") or [PAYLOAD.get("account")] + details = [find_account(token, accounts, str(item)) for item in selectors if item is not None] + if len({item.get("id") for item in details}) != len(details): + raise RuntimeError("account selectors resolve to duplicate accounts") + detail = details[0] if action == "get": return {**base, "ok": True, "operation": "get", "mutation": False, "account": account_summary(detail, True)} - if detail.get("type") != "apikey": - raise RuntimeError("runtime temp-unschedulable policy requires an apikey account") - credentials = dict(detail.get("credentials") or {}) - before = normalize_policy(credentials) - if action == "apply": - template = PAYLOAD.get("selectedTemplate") - if not isinstance(template, dict): - raise RuntimeError("selected runtime template missing") - desired_fields = template.get("credentials") or {} - desired = dict(credentials) - desired.update(desired_fields) - after_plan = normalize_policy(desired) - exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials - change = "noop" if before == after_plan else ("update" if exists else "create") - else: - desired = dict(credentials) - desired.pop("temp_unschedulable_enabled", None) - desired.pop("temp_unschedulable_rules", None) - after_plan = normalize_policy(desired) - change = "delete" if before != after_plan or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop" - include_plan_rules = PAYLOAD.get("full") is True - plan = { - "change": change, - "account": account_summary(detail, include_plan_rules), - "before": policy_summary(before, include_plan_rules), - "after": policy_summary(after_plan, include_plan_rules), - "template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None, - "credentialsPrinted": False, - } - if PAYLOAD.get("confirm") is not True or change == "noop": - return {**base, "ok": True, "operation": action, "mode": "dry-run" if PAYLOAD.get("confirm") is not True else "confirmed-noop", "mutation": False, "plan": plan} - updated = data_of(curl_api("PUT", f"/api/v1/admin/accounts/{detail['id']}", bearer=token, payload={"credentials": desired}), "update account runtime config") - after_detail = updated if isinstance(updated, dict) else account_detail(token, detail["id"]) - return {**base, "ok": True, "operation": action, "mode": "confirmed", "mutation": True, "plan": plan, "account": account_summary(after_detail, include_plan_rules)} + if action == "infrastructure": + return { + **base, + "ok": True, + "operation": "infrastructure", + "mutation": False, + "account": account_summary(detail, False), + "infrastructure": infrastructure_snapshot(token, detail), + } + work = [(item, *runtime_mutation_plan(item, action)) for item in details] + if len(work) == 1: + item, plan, desired = work[0] + if PAYLOAD.get("confirm") is not True or plan["change"] == "noop": + return {**base, "ok": True, "operation": action, "mode": "dry-run" if PAYLOAD.get("confirm") is not True else "confirmed-noop", "mutation": False, "plan": plan} + updated = data_of(curl_api("PUT", f"/api/v1/admin/accounts/{item['id']}", bearer=token, payload={"credentials": desired}), "update account runtime config") + after_detail = updated if isinstance(updated, dict) else account_detail(token, item["id"]) + return {**base, "ok": True, "operation": action, "mode": "confirmed", "mutation": True, "plan": plan, "account": account_summary(after_detail, PAYLOAD.get("full") is True)} + changed = sum(1 for _, plan, _ in work if plan["change"] != "noop") + if PAYLOAD.get("confirm") is not True: + plans = [{**plan, "result": "planned" if plan["change"] != "noop" else "noop"} for _, plan, _ in work] + return {**base, "ok": True, "operation": action, "mode": "dry-run", "mutation": False, "plans": plans, "summary": {"selected": len(work), "changed": changed, "succeeded": 0, "failed": 0}} + succeeded = 0 + failed = 0 + plans = [] + for item, plan, desired in work: + if plan["change"] == "noop": + plans.append({**plan, "result": "noop"}) + continue + try: + data_of(curl_api("PUT", f"/api/v1/admin/accounts/{item['id']}", bearer=token, payload={"credentials": desired}), "update account runtime config") + succeeded += 1 + plans.append({**plan, "result": "succeeded"}) + except Exception as exc: + failed += 1 + plans.append({**plan, "result": "failed: " + str(exc)}) + return {**base, "ok": failed == 0, "operation": action, "mode": "confirmed" if failed == 0 else "confirmed-partial", "mutation": succeeded > 0, "plans": plans, "summary": {"selected": len(work), "changed": changed, "succeeded": succeeded, "failed": failed}} try: result = runtime_result()