From a91238c2006ce757c6096920df93f9c122fbec87 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 06:40:02 +0200 Subject: [PATCH] feat: restore parallel ops and diagnostics changes --- .agents/skills/unidesk-cicd/SKILL.md | 7 +- .agents/skills/unidesk-sub2api/SKILL.md | 11 +- .../skills/unidesk-sub2api/agents/openai.yaml | 4 +- .../unidesk-sub2api/references/codex-pool.md | 55 +- .../references/troubleshooting-accounts.md | 34 + config/platform-infra/sub2api-codex-pool.yaml | 22 +- .../R4.4.2_Task_Report.md | 21 + .../R6_Task_Report.md | 22 + .../R1_Task_Report.md | 37 + .../R2.1_Task_Report.md | 55 + .../R2.2.1_Task_Report.md | 107 ++ .../R2.2.2_Task_Report.md | 64 + .../R2.2.3_Task_Report.md | 53 + .../R2.2.4_Task_Report.md | 104 ++ .../pr-merge-driven-automatic-delivery.md | 2 +- docs/MDTODO/sub2api-upstream-reliability.md | 37 + docs/reference/platform-infra.md | 20 +- scripts/src/output.ts | 4 +- .../remote-python-sync.ts | 126 +- .../platform-infra-sub2api-codex/render.ts | 29 +- .../platform-infra-sub2api-codex/runtime.ts | 1282 ++++++++++++++++- .../src/platform-infra-sub2api-codex/types.ts | 6 +- 22 files changed, 2039 insertions(+), 63 deletions(-) create mode 100644 docs/MDTODO/details/observability-trace-reliability/R4.4.2_Task_Report.md create mode 100644 docs/MDTODO/details/pr-merge-driven-automatic-delivery/R6_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R1_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.1_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.1_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.2_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.3_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.4_Task_Report.md create mode 100644 docs/MDTODO/sub2api-upstream-reliability.md diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 299abf2a..0ad166f6 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -128,12 +128,13 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - CI/CD validation 阶段只能验证部署对象的 `/health` 端点和必要 provenance；禁止在 CI/CD gate 中运行 web-probe、Playwright、远程浏览器截图、用户路径 E2E 或等价重型业务探针。业务/用户入口验证只能作为发布后的独立 post-deploy validation 证据，不得阻塞 CI/CD 一键交付。 - 产品功能配置 schema 的 CI/CD 合同： - repo 内唯一约定路径固定为 `config/feature-config.schema.json`； - - schema 固定使用 JSON Schema Draft 2020-12，并由 lockfile 锁定的完整结构化 validator 校验；禁止用正则、浅层字段遍历或关闭整体 strict 代替； + - schema 固定使用 JSON Schema Draft 2020-12，并由 lockfile 锁定的完整结构化 validator 校验；禁止用正则、只检查顶层字段的浅层解析器或关闭整体 strict 代替； - schema 只描述产品功能配置，不得承载数据 authority、事件路径、投影实现或交付路径选择； - - 每个顶层 `properties` 成员必须声明唯一非空 `x-unidesk-feature`；一个产品功能只允许一个规范配置 property，禁止别名、反向变量或重复开关； + - schema 的每个顶层 property 是一个规范产品配置键，值可以是对象，不强制使用大写 env 命名； + - 每个顶层 property 必须声明唯一非空 `x-unidesk-feature`；同一功能只允许一个规范配置 property，禁止在 schema、renderer、Deployment env 或应用解析器中保留别名、反向变量或重复开关； - candidate snapshot 必须由正常 source-artifact 阶段从 owning YAML 或已渲染 workload 显式生成；禁止读取 runner `process.env` 作为产品配置 authority； - HWLAB、AgentRun 与 Sentinel consumer 必须复用同一 shared helper，并在正常 source-artifact/GitOps publish 路径执行，不得只挂到某个 consumer 的专属后处理； - - CI/CD 必须检查 schema 语法、嵌套配置值匹配和功能到配置 property 的一对一关系，并输出有界 typed warning；输入快照缺失使用独立 typed warning，不得与 schema 非法混淆； + - CI/CD 必须检查 schema 语法、嵌套配置值匹配和功能到配置 property 的一对一关系，并输出有界 typed warning；输入快照缺失使用独立的 `feature-config-input-missing` warning，不得与 schema 非法混淆； - schema 缺失、非法、candidate 缺失、配置不匹配、重复 property、validator/依赖异常或 OTel 导出失败时，只允许记录 warning；embedded 命令必须以 `0` 退出，PipelineRun、artifact、GitOps promote、Argo reconcile、runtime rollout 和 `/health` closeout 必须继续； - schema 校验器只读，禁止写回配置、注入默认值、删除变量、关闭功能、切换架构路径或执行任何补救操作；运行时必须原样接收本次待发布配置； - schema warning 必须实际导出 CI/CD OTel span/event；endpoint、serviceName、sampling 和 exporter timeout 只读 `config/platform-infra/pipelines-as-code.yaml#observability` 与精确 repository params，零/多 repository 匹配在 renderer 生成期 fail-closed； diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index 6c2413ac..0ecfd9cd 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -1,6 +1,9 @@ --- name: unidesk-sub2api -description: UniDesk Sub2API 平台运维技能。用户提到 Sub2API、sub2api、platform-infra sub2api、Codex pool、统一 API key、Sub2API FRP 暴露、Sub2API 管理 UI、配置 master ~/.codex 走 Sub2API、添加/删除 Codex 上游账号、校验 Sub2API /v1/models 时使用。 +description: >- + UniDesk Sub2API 平台运维技能。用户提到 Sub2API、platform-infra sub2api、Codex pool、统一 API key、 + runtime CRUD、精准批量账号操作、临时不可调度、上游错误率、客户可见错误、换号/failover、 + 模型映射、可用模型探测、FRP 暴露、管理 UI、配置 master ~/.codex、上游账号增删或校验 /v1/models 时使用。 --- # UniDesk Sub2API @@ -35,12 +38,14 @@ bun scripts/cli.ts platform-infra sub2api image-prepull --target PK01 --confirm ## 何时读取 reference - 部署、状态、target 边界、PK01 host-Docker、k3s target、egress proxy、镜像升级：读 [references/operations.md](references/operations.md)。 -- Codex pool、统一 key、trace、account temp-unschedulable、`codex-pool sync|validate`：读 [references/codex-pool.md](references/codex-pool.md)。 +- Codex pool、统一 key、runtime CRUD、精准批量、模型探测、trace、account temp-unschedulable、`codex-pool sync|validate`：读 [references/codex-pool.md](references/codex-pool.md)。 - Sentinel、marker-only 判定、账号冻结/恢复、`sentinel-report|sentinel-probe|sentinel-image`：读 [references/sentinel.md](references/sentinel.md)。 - 受保护手动账号代理、分组绑定、WebUI account test：读 [references/manual-accounts.md](references/manual-accounts.md)。 - 添加或删除上游 profile/account：读 [references/upstreams.md](references/upstreams.md)。 - FRP/Caddy、PK01 shared Caddy managed block、public URL 暴露：读 [references/public-exposure.md](references/public-exposure.md)。 - master `~/.codex` 统一消费端配置：读 [references/local-codex-consumer.md](references/local-codex-consumer.md)。 - closeout 验收和最小 smoke：读 [references/validation.md](references/validation.md)。 -- 排障总入口：先读 [references/troubleshooting.md](references/troubleshooting.md)，再按失败层读取运行面、账号池或公网暴露专题。 +- 排障总入口： + - 先读 [references/troubleshooting.md](references/troubleshooting.md)，再按失败层读取运行面、账号池或公网暴露专题。 + - 客户可见错误溯源、错误率分析和配置调优必须继续读 [references/troubleshooting-accounts.md](references/troubleshooting-accounts.md)。 - 禁止事项和越界判断：读 [references/guardrails.md](references/guardrails.md)。 diff --git a/.agents/skills/unidesk-sub2api/agents/openai.yaml b/.agents/skills/unidesk-sub2api/agents/openai.yaml index 32e0717a..c35ac0b3 100644 --- a/.agents/skills/unidesk-sub2api/agents/openai.yaml +++ b/.agents/skills/unidesk-sub2api/agents/openai.yaml @@ -1,7 +1,7 @@ interface: display_name: "UniDesk Sub2API" - short_description: "Operate UniDesk Sub2API and Codex pool" - default_prompt: "Use $unidesk-sub2api to operate the UniDesk Sub2API Codex pool safely." + short_description: "运维 Sub2API、runtime 配置和 Codex 账号池" + default_prompt: "使用 $unidesk-sub2api 通过受控 CLI 运维 Sub2API，分析客户错误，并安全调整 Codex pool runtime 配置。" policy: allow_implicit_invocation: true diff --git a/.agents/skills/unidesk-sub2api/references/codex-pool.md b/.agents/skills/unidesk-sub2api/references/codex-pool.md index c9b06998..26811e27 100644 --- a/.agents/skills/unidesk-sub2api/references/codex-pool.md +++ b/.agents/skills/unidesk-sub2api/references/codex-pool.md @@ -4,11 +4,40 @@ - PK01 host-Docker 的 `sync --confirm` 通过 Sub2API admin API 对齐 group、YAML-managed accounts、统一消费 API key、capacity/loadFactor、WebSocket v2 标记和内置 `temp_unschedulable` 规则； - 统一消费 key 写入 YAML 声明的 `targets[PK01].hostDocker.envPath`； - PK01 路径不创建 k8s Secret、不部署 sentinel 资源，也不触发 `sub2api apply`、Docker compose、Caddy reload 或容器重启； - - `runtime list|get` 查询 pool group 内全部账号或指定账号的实际状态； - - `runtime errors` 按账号汇总 Sub2API 日志中的真实 `account_upstream_error` 状态码，并用 `openai.forward_failed` 提供脱敏错误摘要； - - `runtime apply|delete` 只修改显式 `--account` 指定账号的 `temp_unschedulable` 配置； + - `runtime list|get` 查询 pool group 内全部账号或指定账号的实际状态；`get` 同时披露脱敏 `modelMapping` 和完整临时不可调度规则； + - `runtime models --account` 通过 Sub2API 原生账号模型接口同时读取配置模型和上游实时模型，不把配置映射误当作真实上游能力； + - `runtime errors` 优先使用 Sub2API 原生 admin/ops 能力： + - `--all-groups` 按稳定 group ID 游标分页返回所有分组概要； + - `--group ` 返回指定分组的账号根因、策略效果和 request 索引； + - `--platform ` 可筛选平台；省略分组参数时继续分析 YAML 当前 Codex pool； + - 默认输出使用 Kubernetes 风格紧凑表格；机器读取必须显式使用 `--json`、`--full` 或 `--raw`； + - dashboard 请求量和客户错误率按分组精确统计； + - 账号被多个分组共享时，账号级 upstream-errors 和缺少 `group_id` 的 runtime 事件属于共享证据，禁止跨分组相加； + - dashboard overview 提供请求分母和错误率； + - account availability 提供可用、限流、过载和错误状态； + - concurrency 提供容量与排队； + - upstream errors 提供账号根因； + - 原生 system-log 索引提供事件时直接使用；索引未保留临时不可调度、failover 或 `forward_failed` 时，才从 target runtime 日志补这些事件，并显式披露 fallback 原因和数量； + - `policyEffectiveness` 从同一日志窗口汇总临时不可调度、切号、无候选和网关最终响应，并且只通过 `request_id` 关联切号结果； + - `requestDrilldown` 给出切号后成功、失败、结果缺失和选号失败的有限 request id，可继续交给 `trace` 下钻； + - 只有 `account_temp_unschedulable` 事件才算规则命中，不能仅凭错误状态码推断模板生效； + - `requestIdCoverage` 披露各类事件缺少 request id 的数量；缺失时不能把零关联解释为没有发生摘号后切号； + - `forwardFailureWithHttpSuccessRequestCount` 单列 HTTP 200 与 `forward_failed` 并存的请求，避免把流式响应已提交后的错误误判为完整成功； + - 网关最终错误占比只使用窗口内目标 API 路径的 `http request completed` 日志事件，必须同时披露窗口与 tail 边界，不能解释为完整业务错误率； + - `--full` 额外输出模型、小时、流式分布和有限详情样本； + - 当前并发是查询时快照，不代表错误发生瞬间并发；没有历史并发证据时，命令必须明确输出不可用，不能据此建议降低并发； + - 错误列表没有账号全部路由尝试数时，命令只报告错误数量，不伪造错误率； + - Ops 记录与 runtime 日志是独立证据，命令明确披露采样数量，不按日志位置强行关联； + - `runtime apply|delete` 支持 `temp-unschedulable` 和 `model-mapping` 两类 runtime 配置； + - 单账号必须显式使用 `--account`，精准批量必须显式使用 `--accounts `，不提供隐式 `all`； + - 批量操作先校验全部 selector 存在且账号类型可修改，再顺序调用 Sub2API admin API： + - 选择错误不会产生部分写入； + - 上游 admin API 在批次中途失败时仍需逐账号 `get` 对账； - 写操作不带 `--confirm` 时只返回 dry-run，带 `--confirm` 才通过 Sub2API admin API 写入； - - runtime 命令不输出 API key 或完整 credentials，默认返回紧凑 JSON，使用 `get`、`--full` 或 `--raw` 下钻； + - 批量默认输出账号 ID、名称、change、mode、mutation 和汇总计数；runtime 命令不输出 API key 或完整 credentials，使用 `get`、`--full` 或 `--raw` 下钻； + - 文档中的 runtime action 或参数被当前分支 CLI 拒绝时，按 CLI 集成漂移处理： + - 不退回手写 Sub2API 管理 API。 + - 先确认当前工作分支包含对应受控实现，或完成语义合入后再操作运行面。 - `sentinel-report`、`sentinel-probe`、`sentinel-image` 和部分 `trace` 能力仍以 k3s target 为主，需要这些能力时显式选择对应 k3s target。 k3s 账号池操作示例： @@ -19,10 +48,17 @@ bun scripts/cli.ts platform-infra sub2api codex-pool sync --target D601 --confir bun scripts/cli.ts platform-infra sub2api codex-pool validate --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool runtime list --target PK01 bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --target PK01 --account +bun scripts/cli.ts platform-infra sub2api codex-pool runtime models --target PK01 --account bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h +bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h --all-groups +bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h --group bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --account --template bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --account --template --confirm +bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --accounts --template +bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --accounts --template --confirm bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --target PK01 --account --kind temp-unschedulable +bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --target PK01 --account --kind model-mapping --model --upstream-model +bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --target PK01 --account --kind model-mapping --model bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --target D601 --confirm @@ -41,7 +77,14 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 - `pool.defaultTempUnschedulableTemplate`: YAML-managed 账号默认引用的临时不可调度模板。`sync --confirm` 只把模板渲染到 YAML-managed 账号。 - `profiles.entries[].tempUnschedulableTemplate`: 可选账号级模板引用，只用于明确偏离默认模板。 - YAML 未声明的账号标记为 `runtime-manual`；它可以显示 `matchingTemplate`，但模板差异不是 drift，也不能由 `sync` 自动改写。 -- 手工账号需要采用模板时，必须通过 `runtime apply --account ... --template ... --confirm` 显式写入；需要删除该类配置时使用 `runtime delete --account ... --kind temp-unschedulable --confirm`。 +- YAML 只保存可复用规则模板和 managed account 的模板引用。未在 YAML 预配置的手工账号 runtime 配置必须使用 CRUD，不得为了批量下发而把账号纳入 YAML managed profile。 +- 手工账号需要采用模板时： + - 写入使用显式 `runtime apply --account ...` 或 `--accounts ... --template ... --confirm`。 + - 删除使用相同显式 selector 的 `runtime delete --kind temp-unschedulable --confirm`。 +- 模型映射先用 `runtime models --account ...` 获取原生配置与实时上游能力： + - 只有实时模型列表或真实模型请求证明支持时才确认写入。 + - 禁止把名称相近的模型猜测为别名。 + - 禁止用错误映射掩盖 `model_not_found` 或无候选。 - runtime 模板与外部 `sentinel.*` 分开配置、互不驱动。内置规则负责请求路径即时冷却和 failover；哨兵负责 marker health、账号级隔离、恢复和 probe 退避。 - 外部 sentinel 的写入面只允许通过 Sub2API admin `schedulable` 接口冻结/恢复账号；不能写入、清理或间接清理 `temp_unschedulable_until` / `temp_unschedulable_reason`、rate-limit、overload、model-rate-limit 等 Sub2API 请求路径 runtime 状态，也不能调用 `recover-state` 作为恢复动作。看到 UI 里的“触发时间/解除时间/规则序号/匹配关键词”临时不可调度状态时，默认先归因到 Sub2API 内置 request-path temp-unschedulable，而不是 sentinel。 - YAML 只选择和配置 Codex 上游，不声明 `schedulable` 长期字段；`codex-pool sync --confirm` 不负责把既有账号恢复为 `schedulable=true`。既有账号的 `schedulable=false` 必须由哨兵先同步 Sub2API runtime 状态，再在 marker probe 命中后恢复。 @@ -60,7 +103,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D60 `sync --confirm` 会登录 Sub2API admin、创建/更新 group、创建/更新 YAML 中的 `unidesk-codex-*` accounts，并创建/复用统一 API key。k3s target 还会写入统一 API key Secret 并部署/更新哨兵资源；PK01 host-Docker target 只写 Sub2API runtime 和 host-Docker env 文件。host-Docker `validate` 也先使用 env 中的 Sub2API admin 凭据登录管理 API；如果 env 的统一 `API_KEY` 缺失或不匹配，它只通过管理 API 恢复或创建名为 YAML `pool.apiKeyName` 的统一消费 key 并写回 host-Docker env，不读取本地 profile 文件、不创建/更新账号、不 prune、不改变调度策略。`sync` 不把既有 managed account 直接恢复为 `schedulable=true`。恢复只由哨兵在读取 Sub2API runtime `schedulable=false` 后触发 recovery probe，并在 marker 命中时执行。`sync` 默认不删除 YAML 中缺席的 managed account。只有明确退役上游时才使用 `sync --confirm --prune-removed` 删除缺席且 `extra.unidesk_managed=true` 的 `unidesk-codex-*` account。对 `manualAccounts.protected`，`sync` 只执行 YAML 显式允许的窄同步；当前允许项是从目标 `egressProxy` 创建/更新 Sub2API internal proxy 记录并绑定 `proxy_id`，以及把受保护手动账号加入当前 `pool.groupName`。它仍不接管该账号凭据、status、schedulable、priority/capacity/loadFactor 或哨兵状态。若受保护手动账号在运行面缺失，`sync`/`validate` 会报告 manual account drift；不要自动创建、删除、接管或从 YAML 移除该账号。 -`trace --request-id ` 是只读 request 追溯报表，不触发 probe、不修改账号。默认输出请求开始/最终状态、failover、`account_select_failed`、窗口内 `account_temp_unschedulable`、admin schedulable 写入计数和当前账号快照；`reason=failover-attempted-no-candidate` 表示 Sub2API 已进入自动切号，但排除当前失败账号后没有可用候选。需要机器处理时使用 `--raw`，需要原始匹配行时加 `--show-lines`。 +`trace --request-id ` 是只读 request 追溯报表，不触发 probe、不修改账号。命令优先使用原生 `/admin/ops/requests` 获取请求结果，并使用 `/admin/ops/system-logs` 获取 request-id 索引事件；target runtime 日志只补原生索引未保留的邻近事件。报表显式输出 target、runtime mode、主数据源、上下文数据源和读取状态。默认输出请求开始/最终状态、failover、`account_select_failed`、`forward_failed`、窗口内 `account_temp_unschedulable`、admin schedulable 写入计数和当前账号快照。`forward_failed` 与 HTTP 200 并存且没有 failover 时，结果标记为 `degraded`，不能解释为完整流式成功；`reason=failover-attempted-no-candidate` 表示 Sub2API 已进入自动切号，但排除当前失败账号后没有可用候选。需要机器处理时使用 `--raw`，需要原始匹配行时加 `--show-lines`。 对已支持的 k3s target，`sync --confirm`、`validate` 和 sentinel 操作可能超过单次 SSH/runtime 短连接窗口；远端 job/poll 边界见 [sentinel.md](sentinel.md)。 diff --git a/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md b/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md index eeca3c0b..461a3359 100644 --- a/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md +++ b/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md @@ -1,5 +1,39 @@ ## 账号池排障 +### 客户可见错误与配置调优 + +1. 先用 `runtime errors --since ` 获取全账号同窗口证据： + - 以 Sub2API 原生 dashboard overview 的请求分母、客户错误率和上游错误率为主。 + - 以 account availability、concurrency 和 upstream errors 解释账号可用性、排队、状态码与根因。 + - 原生 system-log 索引缺少策略事件时，才使用 target runtime 日志补临时不可调度、failover、`account_select_failed` 和 `forward_failed`，并保留 fallback 披露。 +2. 从客户可见错误中选有限 request id 运行 `trace`，按请求链区分： + - 上游失败后成功切号。 + - 切号后候选耗尽。 + - 首次选号即无候选。 + - HTTP 200 已提交流式响应后出现 `forward_failed` 的 degraded 请求。 +3. 规则证据必须同时包含真实 HTTP 状态码与响应体关键词： + - 只有状态码、通用包装文案或推断出的原因不能扩展关键词。 + - 只有 `account_temp_unschedulable` 才证明规则命中；不能用相同状态码再次出现来推断规则失效。 + - 优先使用精确短语，避免把 `context window`、认证失败、模型不支持等不可互换错误混入通用不可用规则。 +4. 无候选或模型路由错误先运行 `runtime models`： + - 配置模型来自 `model_mapping`，实时模型来自上游原生同步，两者必须分开报告。 + - 没有账号真实支持目标模型时，保持池不声明该模型并快速失败，不映射到名称相近的其他模型。 + - 上游模型同步失败只能标记证据不足，不能据此补映射。 +5. 调优按所有权下发： + - 通用模板先修改 owning YAML，并保持冷却保守；默认从一分钟开始，没有新证据和明确授权不得超过三分钟。 + - YAML-managed 账号走 `sync`；runtime-manual 账号走显式 `runtime apply|delete`。 + - 多账号使用 `--accounts` 精准集合；例外账号从 selector 中显式排除，不使用隐式全量操作。 + - 先 dry-run 核对 change、before/after 和账号集合，再加 `--confirm`，最后逐账号 `get` 或批量 noop dry-run 对账。 +6. 不把当前并发快照当作错误时并发： + - 只有错误原文明确指向 concurrency，或存在错误时历史并发证据，才把降低上游并发作为有界对照实验。 + - 缺少直接证据时先处理主导上游根因。 +7. 下发后立即拉短窗口只做回归检查： + - 效果判定必须等待新的完整窗口。 + - 同时比较请求分母、客户错误率、上游错误率、规则命中、切号成功、候选耗尽和 degraded 流式请求。 +8. 版本回退必须有源码差异或版本对照证据证明当前版本引入回归： + - 旧版具有同一选择或错误传播逻辑时，不用回退替代配置治理。 + - 新版包含相关 failover 修复时，优先保留新版并治理真实根因。 + - Codex pool 哨兵、账号冻结/恢复、marker-only 判断或 probe 周期看不清：第一步跑 `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report`。这个报表是主观察面；只有报表缺字段或需要底层证据时，才继续看 `--raw`、CronJob log、state ConfigMap 或 Sub2API 管理 UI。若看到“临时不可调度状态”且包含规则序号/匹配关键词，检查 Sub2API `account_temp_unschedulable` 日志和账号 `temp_unschedulable_*` 字段；sentinel 只解释 `schedulable=false` 的 active quarantine，不解释这类内置临时冷却。 - 只加强监控、不让哨兵自动冻结账号时，把 YAML `sentinel.actions.enabled=false` 后 `codex-pool sync --confirm`。此时 marker probe 和 gateway failure monitor 仍记录 `would-freeze` / observe-only 证据，但不会通过 Sub2API admin 写 `schedulable=false`；`/responses/compact` 的 `codex.remote_compact.failed` 和 compact 上游 5xx failover 只作为 `gateway-compact-*` 观察事件记录，不作为哨兵自动切换触发器。 - 单个 request id 报 502/503/中断/没有自动切号：第一步跑 `bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id `。先看 `outcome`、`reason`、`FAILOVER`、`SELECT-FAILED`、`ACCOUNT SIGNALS` 和 `WINDOW STATS`；只有 trace 报表缺字段或需要审计原始日志时，才加 `--show-lines` 或 `--raw`。若 `reason=failover-attempted-no-candidate`，说明切号动作已发生，但 scheduler 在排除失败账号后没有可用候选；继续用 `sentinel-report` 和 `validate --full` 区分 sentinel quarantine、request-path temp-unschedulable、账号 status 或容量耗尽。 diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 31a138f0..9b9702ae 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -12,22 +12,30 @@ runtime: templates: - id: codex-upstream-failover kind: temp-unschedulable - description: Reusable Sub2API request-path cooling and failover policy for Codex-compatible API-key accounts. + description: Codex 兼容 API-key 账号的请求路径短时冷却与换号策略。 spec: enabled: true rules: + - statusCode: 429 + keywords: [concurrency limit exceeded] + durationMinutes: 1 + description: 明确命中上游并发限制时短时冷却当前账号。 + - statusCode: 500 + keywords: [failed to validate api key, do request failed] + durationMinutes: 1 + description: 上游 API-key 查询依赖或请求传输内部失败时短时冷却当前账号。 - statusCode: 502 - keywords: [upstream service temporarily unavailable, upstream request failed, context window, stream usage incomplete, missing terminal event] + keywords: [upstream service temporarily unavailable, upstream request failed, service temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 - description: Observed 502 upstream availability, context-window, and incomplete-stream failures briefly cool down the selected account. + description: 明确命中 502 上游不可用、过载或并发限制时短时冷却当前账号。 - statusCode: 503 - keywords: [upstream service temporarily unavailable, upstream request failed, temporarily unavailable] + keywords: [upstream service temporarily unavailable, upstream request failed, service temporarily unavailable, temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 - description: Observed 503 temporary upstream availability failures briefly cool down the selected account. + description: 明确命中 503 上游暂时不可用、过载或并发限制时短时冷却当前账号。 - statusCode: 524 - keywords: [upstream request failed, missing_required_parameter, encrypted_content] + keywords: [upstream request failed, service temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 - description: Observed 524 upstream request and encrypted-content failures briefly cool down the selected account. + description: 明确命中 524 上游不可用、过载或并发限制时短时冷却当前账号。 pool: groupName: unidesk-codex-pool diff --git a/docs/MDTODO/details/observability-trace-reliability/R4.4.2_Task_Report.md b/docs/MDTODO/details/observability-trace-reliability/R4.4.2_Task_Report.md new file mode 100644 index 00000000..1b6248bf --- /dev/null +++ b/docs/MDTODO/details/observability-trace-reliability/R4.4.2_Task_Report.md @@ -0,0 +1,21 @@ +# R4.4.2 任务报告 + +## 当前状态 + +任务仍在进行中。UniDesk #1959 / PR #1961 已解决 runtime GitOps 大段脚本内联导致的 ARG_MAX 问题，后续 PR 已修复 ConfigMap ownership、YAML 解析和 terminal marker 组合；这些修复保留。 + +## 最新架构纠偏 + +此前把 direct/live/refresh/projector/outbox/realtime 六个架构布尔值同时投影到 YAML、renderer、Deployment env、应用 parser 和 CI verify，已经造成合法自动链被错误阻塞，并使缺少单个 env 的新 cloud-api Pod CrashLoop。该模型已废弃： + +- 数据 authority 与事件路径固定为 agentrun.event.v1 → transactional projector → PostgreSQL facts/outbox → hwlab.event.v1 → realtime/replay SSE。 +- 架构不再提供 env/YAML 开关，旧 HWLAB #2536 已关闭。 +- 产品功能使用仓库内 config/feature-config.schema.json，一个功能只对应一个规范变量。 +- schema 缺失、非法、不匹配或重复只产生 typed warning；不得写回配置、关闭功能、切换路径或阻塞滚动上线。 + +## 当前执行 + +- UniDesk #1981 / MDTODO R6：删除 CI/CD 六 env 硬门禁并实现只读 warning schema 校验；Artificer task qt_7d73811bd7194ea186c904645da8be6c。 +- HWLAB #2538 / MDTODO R4.4.2.10：删除运行时架构 env 和旧分支，固定纯 Kafka transactional 投影；Artificer task qt_37898861dcea44219115c91075af340f。 + +任务将在 PR 审核、正常 merge 自动交付、NC01/v03 新 Pod Ready、Argo Synced/Healthy、Kafka 健康指标归零及新 hi 实时/回放/canonical duration 验收后收口。 diff --git a/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R6_Task_Report.md b/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R6_Task_Report.md new file mode 100644 index 00000000..9519de76 --- /dev/null +++ b/docs/MDTODO/details/pr-merge-driven-automatic-delivery/R6_Task_Report.md @@ -0,0 +1,22 @@ +# R6 任务报告 + +## 结论 + +- 产品功能 schema 固定为 `config/feature-config.schema.json`，使用 JSON Schema Draft 2020-12 与锁定的 `ajv-dist@8.17.1` 结构化 validator。 +- 每个顶层 property 通过唯一 `x-unidesk-feature` 建立一对一配置关系；CI runner env 不作为产品配置 authority。 +- schema、输入、重复、值不匹配、validator 与 OTel 异常均只产生 typed warning，不写回配置、不关闭功能、不切换架构，也不阻断 artifact、GitOps、Argo、rollout 与 health。 +- PR #1987 完成 schema/validator/OTel/`status|history|debug-step` 投影和旧 capability env 门禁删除；PR #1989 修复 source artifact 缺少 Ajv bundle 的首断点。 + +## 验证 + +- 定向测试首轮 53/53；主代理 shell 状态纠偏后回归 49/49。 +- merge commit `92b19ce6c8b55b1059506d6b4527388d1f671828` 自动创建 `unidesk-host-v2xn8`，44 秒 Succeeded。 +- GitOps `fa4d36a288ac2943091ffb92e0a4d5a0e93e5ff5`；Argo `Synced/Healthy` 且 revision relation=exact。 +- registry/runtime digest 同为 `sha256:a10a22b94976eb65a664b0b3b0c838a5720d39c037229b325c6fe0083578f355`，1/1 ready，health=200。 +- 全程未人工创建 PipelineRun、mirror sync、Argo refresh 或 patch 运行面。 + +## 工件 + +- Issue: https://github.com/pikasTech/unidesk/issues/1981 +- PR: https://github.com/pikasTech/unidesk/pull/1987 +- Follow-up PR: https://github.com/pikasTech/unidesk/pull/1989 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R1_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R1_Task_Report.md new file mode 100644 index 00000000..2cbe3888 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R1_Task_Report.md @@ -0,0 +1,37 @@ +# R1 任务报告 + +## 目标 + +把 Sub2API 上游故障调查从零散日志读取收敛为可复用的受控 CLI 能力，并用实际错误证据形成保守、可逆的账号级 runtime 调度缓解。 + +## 工具改进 + +- `runtime list|get` 补充 `proxyId`、并发上限、当前并发、`loadFactor` 和优先级，能够直接核对代理与调度事实。 +- `runtime errors` 综合 `account_upstream_error`、`openai.forward_failed` 和 Sub2API Ops 上游错误记录，输出全池主因排名、账号主因、状态码、消息、模型、小时和流式分布；详细样本通过 `--full` 渐进披露。 +- 错误分析显式区分 Ops 样本与 runtime 日志，不按时间或位置强行关联；缺少全部请求尝试分母时不计算错误率，缺少错误发生时并发历史时不把当前并发快照作为降并发依据。 +- `runtime apply|delete` 仅修改显式指定账号的临时不可调度配置；无 `--confirm` 时为 dry-run，有 `--confirm` 时才写入，并回读模板匹配、代理、并发、状态和调度状态。 + +## 全池结论 + +- 最近观测窗口内主要错误是仅状态码上游失败、通用上游失败、过载和上游 API-key 校验失败；明确并发错误只占较小部分，降低全池并发量不应作为首要修复。 +- 账号 27 的集中 HTTP 500 证据为 `Failed to validate API key`，指向上游网关 key 查询或依赖失败；账号 32 才有明确并发限制证据。 +- context window、缺少 terminal event 和 `encrypted_content` 属于请求语义或流式续链问题，不应通过临时摘号模板泛化掩盖。 +- Sub2API 仅能在响应尚未提交时通过临时不可调度触发换号；SSE 已向客户端写出后再缺少终止事件，不能透明切换账号。 + +## 配置与运行面应用 + +- owning YAML 的 `codex-upstream-failover` 模板收敛为 5 条实际观测规则：`429`、`500`、`502`、`503`、`524`，每条冷却 1 分钟。 +- `500` 仅匹配实际证据中的 `failed to validate api key` 和 `do request failed`；可用性、过载和并发关键词按实际状态码分别配置。 +- 通过 runtime CRUD 将模板显式应用到 9 个 API-key 手工账号；OAuth 账号 `lyon9801` 保持 `enabled=false`、零规则。 +- 未改变账号凭据、代理、并发上限、优先级、长期 `status` 或 `schedulable`；原先已禁用或错误的账号保持原状。 + +## 验证 + +- 全池 runtime 回读确认 9 个 API-key 账号均为 5 条规则并匹配模板，OAuth 例外未变化。 +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts` 通过。 +- Bun 内置 YAML 解析确认规则状态码为 `429,500,502,503,524`，持续时间均为 1 分钟。 +- `git diff --check` 通过。 + +## 后续边界 + +本次只能证明配置正确应用，不能立即证明错误率下降。后续应在独立的新时间窗口复核错误记录、切号证据和最终客户响应，并继续保持错误数量、请求分母和流式提交阶段的语义区分。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.1_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.1_Task_Report.md new file mode 100644 index 00000000..e494a8bc --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.1_Task_Report.md @@ -0,0 +1,55 @@ +# R2.1 任务报告 + +## 目标 + +增强 UniDesk Sub2API CLI，优先使用 Sub2API 原生 admin/ops 运维能力诊断全池错误、账号状态、并发和单请求链路，运行日志只补原生索引未提供的策略事件。 + +## 官方能力依据 + +对照 `Wei-Shaw/sub2api` 官方仓库提交 `5d1c577cb2c735ca1f1d57533dff1302f6998d91`，确认可复用接口： + +- `/api/v1/admin/ops/dashboard/overview`：请求总数、客户可见错误、错误率、上游错误率、健康分。 +- `/api/v1/admin/ops/upstream-errors`：账号级上游错误、状态码、request id 和详情。 +- `/api/v1/admin/ops/account-availability`：账号可用、限流、过载和错误状态。 +- `/api/v1/admin/ops/concurrency`：平台、组和账号并发占用、容量与排队。 +- `/api/v1/admin/ops/requests`：成功与错误请求的统一 request-id 视图。 +- `/api/v1/admin/ops/system-logs` 及 `/health`：索引日志和 sink 健康状态。 + +## CLI 改进 + +- `runtime errors` 增加 `nativeOps`，使用 overview 的完整请求分母计算客户可见错误率，避免把错误记录数当错误率。 +- 增加原生账号可用性、组并发、排队和 system-log sink 丢弃/写入失败状态。 +- Ops 上游错误改为所有选中账号主动查询，不再要求容器日志先出现对应事件。 +- `policyEffectiveness` 关联临时不可调度、failover、选号失败、`forward_failed` 和最终结果；仅按 request id 关联并披露缺失覆盖率。 +- 原生 system-log 索引没有策略事件而 overview 存在上游错误时，自动回退 target runtime 日志补事件，并披露 fallback 原因、数量和原生索引计数。 +- 全池默认只输出主因排名，账号详情通过 `--account` 下钻；全池和单账号默认输出均保持在 CLI 预算内，`--full` 保留完整分布。 +- `trace` 改为 `/ops/requests` 与 `/ops/system-logs` 优先，target runtime 日志只补邻近上下文。 +- 修复 PK01 trace 固定读取 k3s Deployment、Docker stderr 业务日志遗漏和原生事件倒序问题。 +- `forward_failed` 与 HTTP 200 并存且没有 failover 时，trace 标记为 `degraded`，不再误报普通成功。 +- 修正 `No available channel for model` 根因分类和所有 Ops 错误被描述为 HTTP 500 的误导文案。 + +## 现场验证 + +- 10 分钟原生 overview 样本曾观测 165 个请求、2 个客户可见错误，错误率 1.21%，上游错误率 3.03%；后续窗口为 189 个请求、1 个客户可见错误，错误率 0.53%，上游错误率 2.65%。这些是滚动窗口快照，不代表长期趋势。 +- 原生 concurrency 显示组容量 218，现场占用约 9 至 10，无排队；全池错误不能归因于 Sub2API 组并发耗尽。 +- 原生 availability 显示账号 29、30 因 `403 Insufficient account balance` 处于 error；账号 27 在短时冷却窗口可能暂不可用。 +- 策略事件样本显示临时不可调度和 failover 均实际发生；部分切号成功，仍存在切号后失败、结果缺失和 `forward_failed + HTTP 200`。 +- request `03a94444-ff26-4038-b1b9-67e249ce7f59`：账号 33 返回 524 后切到账号 31，账号 31 返回 502，随后 `account-select-failed/context canceled`，最终客户状态 502。 +- request `55d32c6a-09ae-46ac-9a34-3f63934e7f62`：账号 31 出现并发错误但 HTTP 200，没有 failover，trace 正确标记 `degraded`。 +- 账号 31 的 10 分钟 Ops 记录同时包含 429 rate-limit 和明确 concurrency 文案；当前并发快照不能替代错误发生时并发，后续只能通过受控对比判断是否需要调整账号并发。 +- 原生 system-log sink 健康，无 dropped/write failed；但策略事件查询为零，当前需要 runtime 事件 fallback。 + +## 验证命令 + +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts` +- `bun --check scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts` +- `bun --check scripts/src/platform-infra-sub2api-codex/render.ts` +- `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 10m` +- `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 10m --account 31` +- `bun scripts/cli.ts platform-infra sub2api codex-pool trace --target PK01 --request-id 03a94444-ff26-4038-b1b9-67e249ce7f59` +- `git diff --check` + +## 后续 + +- R2.2 持续观察新窗口，区分客户可见错误率、上游错误率、成功 failover 和流式 degraded。 +- R2.3 调查原生 system-log 索引为何未保留策略事件，并优先推动 Sub2API 原生能力覆盖，减少 runtime 日志 fallback。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.1_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.1_Task_Report.md new file mode 100644 index 00000000..ba555db9 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.1_Task_Report.md @@ -0,0 +1,107 @@ +# R2.2.1 客户可见错误溯源与调优建议 + +## 范围与数据源 + +- 运行目标：PK01 host-Docker，Sub2API `0.1.153`。 +- 原生 API：`dashboard/overview`、`upstream-errors`、`account-availability`、`concurrency`、`requests`、`system-logs/health`。 +- 原生 `system-logs` 未索引策略事件，按既定 CLI 规则回退读取 runtime 日志。 +- 观察窗口：连续 20 分钟快照与 60 分钟汇总。 +- 本轮只分析，没有修改 YAML、runtime 配置或 Sub2API 代码。 + +## 客户错误率 + +- 20 分钟窗口：335 次请求，4 次客户错误，客户错误率 1.19%，上游错误率 2.99%。 +- 60 分钟窗口：907 次请求，8 次客户错误，客户错误率 0.88%，上游错误率 3.64%。 +- 60 分钟内发生 7 次 `account_select_failed`，几乎覆盖全部客户可见错误。 +- 组并发快照为 9/218，负载 4.13%，排队 0；全池并发不是主因。 +- system-log sink 的 queue、drop 和 write failure 均为 0；日志落库不是业务错误来源。 + +## 请求级根因 + +### 模型候选为零 + +- 三个 `gpt-5.4-nano` 流式请求分别在 5ms、10ms、28ms 内未选中账号并返回 404。 +- 请求 ID：`21e5d449-3b0d-4a70-a798-1dc0af0ba110`、`55f7eb2a-b469-4d90-9ec9-3d20d278aef7`、`e90dfd63-22c1-419b-a479-53f860d4151a`。 +- 官方源码会在组内存在账号但没有任何 `model_mapping` 支持请求模型时返回 `404 model_not_found`。 +- 这不是瞬时容量问题，重试、临时不可调度和提高切号次数均无效。 + +### 多账号连续失败后候选耗尽 + +- `8150234d-eee2-4a08-a43e-e47a22274609`：`gpt-5.4`，account 28 返回 503、account 27 返回 429、account 31 返回 429，约 6.1 秒后无候选，最终 429。 +- `bf9aea0a-869a-4ee0-a99c-3b0ceb553822`：`gpt-5.6-luna`，account 28 返回 503、account 27 返回 429，约 67 秒后上下文取消且无候选，最终 429。 +- `03a94444-ff26-4038-b1b9-67e249ce7f59`：`gpt-5.6-sol`，account 33 返回 524、account 31 返回 502，约 180 秒后上下文取消且无候选，最终 502。 +- `5d8f05ff-9cbc-4040-b6eb-4b262ab85f39`：`gpt-5.6-sol`，account 33 返回 524，约 126 秒后上下文取消且无候选，最终 502。 +- `max_account_switches=10` 只是上限。官方实现会排除已失败账号并重新选号；剩余账号不满足模型或运行状态时，会在只切 1 至 3 次后结束。 + +### 流式提交后错误 + +- 成功样本证明自动 failover 正常工作：account 27 的 502 可切到 account 33，account 28 的 503 可切到 account 31。 +- 存在 `forward_failed + HTTP 200`，例如 account 31 返回 `Concurrency limit exceeded`，但下游已经提交流式响应。 +- 官方 `v0.1.153` 在检测到已写入语义响应后明确禁止换号，以免拼接两个上游流。 +- 此类错误不能靠临时不可调度透明恢复，只能减少上游发生率，或由客户端在流中断后重试或恢复。 + +## 上游账号主因 + +- 60 分钟上游记录共 72 条：rate-limit 24、overload 15、concurrency 12、generic upstream failure 11、status-only 6、model-route-unavailable 4。 +- account 28：10 次 overload，当前高频触发 503 临时不可调度。 +- account 27：rate-limit 为主，同时有 502 generic failure；现有规则只覆盖 502，不覆盖真实 429 文本。 +- account 31：13 次 rate-limit 为主，另有 concurrency 和一次 `GROUP_DISABLED`；模板未匹配 `Upstream rate limit exceeded`。 +- account 19：7 次 504，其中 2 次明确 concurrency、1 次 upstream unavailable、4 次只有状态；另有 3 次 `No available channel for model`。 +- account 32：2 次 429 rate-limit、1 次 502 concurrency。 +- account 29、30：余额不足导致 403，已处于 `status=error`、`schedulable=false`，不会继续被调度，但减少了候选容量。 + +## 官方版本判断 + +- PK01 运行 `v0.1.153`。 +- `v0.1.152..v0.1.153` 的相关变更是 WebSocket ingress 生命周期限制，不改变当前 HTTP 模型选号、failover 或临时不可调度匹配。 +- `v0.1.149` 已包含无账号支持模型时返回 404 的实现，因此回退不会消除 `gpt-5.4-nano` 404。 +- `v0.1.153` 已包含 compact 心跳不阻断 failover 的官方 #3887 修复。没有证据表明回退能降低当前错误率，不建议回退。 + +## 调优建议 + +### P0：修正模型目录与映射 + +- 查明 `gpt-5.4-nano` 是客户端误配、错误广告模型还是预期别名。 +- 若没有真实上游支持，从客户可见模型目录和调用配置中移除。 +- 若业务确实需要，只映射到已经实测支持的真实模型，不能为消除 404 添加虚假映射。 + +### P1：扩展保守临时不可调度模板 + +- 在 429 规则增加真实关键词 `upstream rate limit exceeded`，保持 1 分钟冷却。 +- 新增 504 规则时只加入实测关键词 `concurrency limit exceeded` 和 `upstream service temporarily unavailable`，保持 1 分钟冷却。 +- 不增加任意 504 规则；4 次 504 只有状态码，不能推断真实原因。 +- 不把 `No available channel for model` 放入通用账号级模板；这是模型级缺口，账号级摘除会误伤其他模型。 +- `lyon9801` 继续保持例外，不绑定临时不可调度规则。 + +### P1：账号级并发对照调优 + +- 不降低组级并发。 +- account 31 有最强直接并发证据，可先做单账号对照，例如从 30 降到 20，观察 30 至 60 分钟的错误和吞吐。 +- account 19、27、32 也有信号，但没有错误时刻并发历史；不要一次性全降。 +- 手工账号属于 runtime 管理面，后续通过 runtime CRUD 调整，不纳入 YAML managed profile 自动同步。 + +### P1：降低慢失败对客户时间预算的吞噬 + +- 524 请求分别在约 126 秒和 180 秒后穿透，第一个慢上游已消耗大部分客户超时预算。 +- 官方默认 `gateway.response_header_timeout=600s`，明显长于当前客户或边缘上下文。 +- 建议先确认运行实际值，再受控收紧 OpenAI 响应头超时，为一次 failover 预留预算；不能直接照抄固定值。 +- 对照指标应包含首 token 延迟、超时型上游错误、failover 成功率和客户错误率。 + +### P2：恢复候选容量与调度偏好 + +- account 29、30 只有补足余额并通过真实请求验证后才能恢复，不能直接改为 schedulable。 +- account 28、27、31 当前优先级高且错误集中，可在确认模型覆盖后降低调度偏好，让更稳定账号先承担请求。 +- Ops system-log 未索引策略事件，60 分钟有 144 条事件依赖 runtime fallback；这不造成业务错误，但限制长期归因，由 R2.3 跟踪。 + +## 不建议的措施 + +- 不建议回退到 `v0.1.149`。 +- 不建议提高 `max_account_switches`。 +- 不建议降低全池并发。 +- 不建议启用 `pool_mode` 做同账号重试。 +- 不建议对所有 4xx/5xx 做无关键词临时不可调度。 + +## 后续验收 + +- 按 20 分钟和 60 分钟双窗口继续观察客户错误率、upstream error rate、`account_select_failed`、模型 404、failover 成功率、`forward_failed + 200` 和每账号根因。 +- 每次只改变一个变量并保留前后窗口；R2.2 继续保持进行中。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.2_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.2_Task_Report.md new file mode 100644 index 00000000..9d7afedc --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.2_Task_Report.md @@ -0,0 +1,64 @@ +# R2.2.2 任务报告 + +## 目标 + +执行三项客户错误调优： + +1. 处理 `gpt-5.4-nano` 无候选。 +2. 为 429 临时不可调度规则加入真实上游关键词 `upstream rate limit exceeded`。 +3. 增加仅匹配 `concurrency limit exceeded` 与 `upstream service temporarily unavailable` 的 504 规则。 + +同时按用户追加要求增强精准批量 runtime API。 + +## 模型能力结论 + +通过 Sub2API 原生账号接口 `GET /api/v1/admin/accounts/:id/models` 和 `POST /api/v1/admin/accounts/:id/models/sync-upstream` 检查账号 15、19、27-34： + +- 所有账号当前 `model_mapping` 均不包含 `gpt-5.4-nano`。 +- 可实时同步的账号 19、27、28、32、33、34 的上游模型列表均不包含该模型。 +- 账号 29、30、31 的上游模型同步返回 HTTP 403，不能据此声明支持。 +- `lyon9801` 为 OAuth 账号，Sub2API 原生同步接口明确不支持该类型，当前映射也不包含该模型。 + +因此没有创建虚假的 `nano -> mini` 或同名映射。当前池的 `/v1/models` 不宣称支持 `gpt-5.4-nano`，直接请求继续快速返回无候选 404。后续只有真实上游模型列表或实测请求证明支持时，才通过显式 model-mapping CRUD 加入。 + +## 配置调优 + +YAML 模板 `codex-upstream-failover` 保持一分钟冷却，并累计为六条状态规则： + +- 429：`concurrency limit exceeded`、`upstream rate limit exceeded`。 +- 500：`failed to validate api key`、`do request failed`。 +- 502、503、524：保留前序已验证的上游不可用、过载和并发关键词。 +- 504：仅 `concurrency limit exceeded`、`upstream service temporarily unavailable`。 + +通过受控 runtime CLI 对账号 19、27-34 dry-run 后确认下发。逐账号回读均为 `ruleCount=6`、状态码 `429/500/502/503/504/524`，且匹配模板。账号 15 `lyon9801` 保持 `ruleCount=0`，没有应用临时不可调度。 + +## CLI 增强 + +新增： + +- `runtime models --account `：同时显示脱敏 `model_mapping`、配置模型和原生上游实时模型。 +- `runtime apply|delete --kind model-mapping`：显式单账号模型映射 CRUD，默认 dry-run。 +- `runtime apply|delete --accounts `：一次受控作业内按显式账号集合顺序执行，拒绝隐式 all、重复账号和非 API-key 账号。 +- 批量默认输出压缩为账号 ID、名称、change、mode、mutation 和汇总计数；`--full/--raw` 保留完整计划。 + +批量验收命令精确选择账号 `19,27,28,29,30,31,32,33,34`，结果为 `requestedCount=9`、`succeededCount=9`、`mutatedCount=0`，全部 noop，且未包含账号 15。 + +## 即时观察 + +调优后拉取 20 分钟 runtime 错误窗口： + +- 匹配 12 个日志事件。 +- 结构化上游状态为 502 一次、524 六次。 +- 账号 27 仍出现四次并发限制文本，但该窗口跨越配置下发时刻，不能用于评价新规则最终效果。 +- 账号 33 有一次 `gpt-5.6-sol` 与 ChatGPT Codex 账号不兼容的 400，属于独立模型路由问题。 +- 未观察到配置写入造成的新类型异常。 + +父任务 R2.2 继续用后续完整窗口观察客户可见错误率、切号和冷却命中效果。 + +## 验证 + +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts`：通过。 +- `git diff --check`：通过。 +- 九账号 runtime dry-run、confirm、逐账号 get：通过。 +- 精准批量 dry-run：通过。 +- 20 分钟 runtime errors：通过。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.3_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.3_Task_Report.md new file mode 100644 index 00000000..e0c9dc3c --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.3_Task_Report.md @@ -0,0 +1,53 @@ +# R2.2.3 任务报告 + +## 目标 + +把本轮形成的 Sub2API runtime 控制、原生数据诊断、客户可见错误溯源和配置调优方法沉淀到 `unidesk-sub2api` skill，避免后续代理重新试探入口、证据口径和下发边界。 + +## 沉淀内容 + +### Skill 触发与路由 + +- 扩展 skill description，使 runtime CRUD、精准批量、临时不可调度、上游错误率、客户可见错误、failover、模型映射和可用模型探测均能触发本 skill。 +- `SKILL.md` 将 runtime 控制路由到 `references/codex-pool.md`。 +- 客户错误溯源、错误率分析和配置调优路由到 `references/troubleshooting-accounts.md`。 +- 同步更新 `agents/openai.yaml` 的简述和默认提示。 + +### Runtime 控制合同 + +`references/codex-pool.md` 固化： + +- `runtime list|get|models|errors|apply|delete` 的职责。 +- `get` 的脱敏 `modelMapping` 与临时不可调度规则可见性。 +- `models` 区分配置模型和上游实时模型。 +- `temp-unschedulable` 与 `model-mapping` 两类 CRUD。 +- 单账号 `--account` 和显式精准批量 `--accounts`，禁止隐式 `all`。 +- 批量选择预检、顺序写入、中途 admin API 失败后的逐账号对账要求。 +- 默认 dry-run、`--confirm` 写入、紧凑批量输出和 Secret/credentials 脱敏。 +- YAML 只保存通用模板和 managed 引用，runtime-manual 账号继续使用 CRUD，不为批量操作强行纳入 YAML。 +- 文档命令与当前分支 CLI 不一致时按集成漂移处理，禁止退回手写管理 API。 + +### 调优分析流程 + +`references/troubleshooting-accounts.md` 固化八步流程： + +1. 用原生 dashboard、availability、concurrency、upstream errors 获取全账号同窗口事实。 +2. 用有限 request id 和 `trace` 区分切号成功、候选耗尽、首次无候选和流式 degraded。 +3. 临时不可调度规则必须同时有真实状态码与响应关键词证据。 +4. 无候选先用原生模型接口核实能力，不猜测近似模型映射。 +5. 按 YAML-managed 与 runtime-manual 所有权分别下发，默认一分钟冷却且无新证据不超过三分钟。 +6. 不用当前并发快照证明错误时并发，降低并发只做有直接证据的有界对照。 +7. 短窗口只做回归，效果等待新的完整窗口并综合请求分母、错误率、规则命中和切号结果。 +8. 版本回退必须有源码差异或版本对照证据。 + +## 实现边界 + +精准批量和 `runtime models` 已在 `/root/unidesk/.worktree/sub2api-error-tuning` 完成真实 PK01 验证。主工作区当前 runtime CLI 尚未语义吸收这部分实现，且同文件存在并行诊断增强修改，因此本任务没有覆盖主工作区 runtime 源码。skill 已加入 CLI 集成漂移防护，后续必须先完成语义合入，不能手写 Sub2API admin API 绕过。 + +## 验证 + +- `quick_validate.py .agents/skills/unidesk-sub2api`：通过。 +- `git diff --check`：通过。 +- skill 主文件 46 行，三个核心文件合计 207 行，保持渐进披露。 +- `agents/openai.yaml` 的 `default_prompt` 显式引用 `$unidesk-sub2api`。 +- 未写入账号 ID、供应商名称、当前错误率或其他一次性运行数据。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.4_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.4_Task_Report.md new file mode 100644 index 00000000..a523f848 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.4_Task_Report.md @@ -0,0 +1,104 @@ +# R2.2.4 所有分组客户错误与 CLI 渐进披露报告 + +## 交付范围 + +- `runtime errors --all-groups`：按稳定 group ID 游标分页，默认返回所有分组概要。 +- `runtime errors --group `：下钻单个分组的原生 Ops 指标、账号根因、策略效果和 request ID 索引。 +- `--platform `：可选平台过滤。 +- `--page-token `：固定分页游标，不使用手工 `limit` 规避输出问题。 +- `runtime list|get|errors|apply|delete`：默认统一为 Kubernetes 风格紧凑表格。 +- `--json`、`--full`、`--raw`：仅在显式请求时返回机器结构或完整披露。 + +## 运行面证据 + +- 目标：PK01 host-Docker。 +- 实际镜像：`docker.1panel.live/weishaw/sub2api:0.1.153`，容器 healthy。 +- 数据源：Sub2API 原生 dashboard overview、account availability、concurrency、upstream-errors、requests 和 system-logs。 +- 原生 system-log 没有保留全部策略事件时，CLI 显式回退到 runtime 日志。 +- system-log sink 的 queue、drop 和 write failure 均为 0。 + +## 所有分组概要 + +同一 60 分钟查询页共 3 个分组： + +- 全局：757 次请求，8 次客户错误，客户错误率 1.06%，34 次上游错误，上游错误率 4.49%。 +- `unidesk-codex-pool`，group 2：277 次请求，6 次客户错误，错误率 2.17%；21 次上游错误，上游错误率 7.58%。 +- `自用`，group 3：480 次请求，2 次客户错误，错误率 0.42%；13 次上游错误，上游错误率 2.71%。 +- `grok`，group 5：窗口内无请求和错误。 +- group 2 承担 36.6% 请求，却产生 75% 客户错误，是当前主要客户错误来源。 + +查询时两个活跃分组均为 3/218 并发、队列 0。降低全局或分组并发不能解释当前错误，不应作为主修复。 + +## 账号和共享证据边界 + +- group 2、3 共享大部分 OpenAI 账号。 +- 原生 dashboard 请求量和客户错误率按 group 精确统计，可以跨组汇总。 +- upstream-errors 是账号级数据；共享账号的根因记录会出现在多个 group 下钻中，禁止跨组相加。 +- 缺少 `group_id` 的临时摘号事件只能按账号 membership 和 request ID 关联，CLI 已标注为共享证据。 +- 原生聚合显示每个活跃组 9 个账号、6 个可用、3 个错误账号。 +- 错误账号为 29、30 的余额不足 403，以及 31 的 API Key 所属分组已删除 403。 + +账号共享窗口内的 34 条上游根因记录为： + +- overload：18。 +- status-only-upstream-failure：9。 +- generic-upstream-failure：7。 + +## 客户可见错误链 + +### 长耗时失败后 context 已取消 + +- `cf6b6257-9640-414f-918d-c55180fb6a8d`：`gpt-5.6-luna`，账号 27 返回 502，约 41.6 秒后进入 failover，下一次选号立即报 `context canceled`，最终 502。 +- `4ee54daa-f41c-4f50-bd72-11080c38e299`：`gpt-5.6-sol` 流式请求，账号 27 返回 502，约 730.8 秒后进入 failover，下一次选号立即报 `context canceled`，最终 502。 +- 两条请求均已触发切号；失败点是第一个上游耗尽了客户请求时间预算，不是 `max_account_switches` 太小，也不是池容量不足。 + +### HTTP 200 后流式失败 + +- `46c1374a-c7f1-49f8-877d-d7ffb216976c`：账号 28 返回 503 后切到账号 33，网关最终 access status 为 200。 +- 随后出现 `openai.forward_failed`，原文指向 `input[2].encrypted_content` 的 ObjectParam/required-parameter 错误。 +- CLI 现判定 `outcome=degraded`、`reason=forward-failed-http-success`，不再把 HTTP 200 误报为完整成功。 +- 流式语义已经提交后不能安全拼接另一个上游响应，透明换号只适用于提交语义响应之前。 + +## 临时不可调度和 failover 效果 + +- group 3 下钻观察到 20 次临时不可调度事件、13 个 failover 请求。 +- 其中 2 个明确成功、1 个明确失败、10 个结果在有限日志索引中缺失。 +- 另有 1 个 `forward_failed + HTTP 200` degraded 请求。 +- 规则能减少立即重复命中坏账号，但无法恢复已经耗尽 context 的请求，也无法恢复已经提交语义流后的协议错误。 + +## 官方仓库与版本判断 + +- 官方 v0.1.153 发布说明只涉及调度缓存异常时间、WebSocket 生命周期等修复，没有声明改变当前 HTTP Responses 的 failover 时间预算或流式提交后切号规则。 +- 官方 issue #2067 与合并 PR #2068 证明旧版曾存在 Responses 多轮 reasoning item 回放导致 404/502 的协议转换缺陷；修复已在 2026-04-29 合入主线。 +- 当前 `encrypted_content` 原文与该 issue 相近但不完全相同，不能直接判定为同一个缺陷。 +- 现有证据不支持从 v0.1.153 回退；回退可能重新引入已修复的 Responses 协议问题。 +- 官方来源： + - https://github.com/Wei-Shaw/sub2api/releases/tag/v0.1.153 + - https://github.com/Wei-Shaw/sub2api/issues/2067 + - https://github.com/Wei-Shaw/sub2api/pull/2068 + +## 调优建议 + +- P0：保持 v0.1.153，不回退，不提高 `max_account_switches`，不降低全池并发。 +- P0：确认实际 OpenAI 响应头/首 token 超时，把单个慢上游的允许时间收紧到能为至少一次 failover 留出预算；先做有界对照，不直接照抄固定值。 +- P1：继续使用最多 3 分钟的保守临时不可调度；当前一分钟冷却已经频繁命中，不扩大到无关键词的所有 5xx。 +- P1：针对账号 28 的 overload 和账号 27 的 502 保持即时冷却，观察完整 60 分钟窗口的客户错误率和 failover 成功率。 +- P1：对账号 33 的 `gpt-5.6-sol` continuation/`encrypted_content` 错误积累更多 request 证据；重复出现时再做账号与模型级调度隔离，不能凭单个样本做全账号摘除。 +- P1：修复账号 29、30 的余额和账号 31 的失效 API Key 后，通过真实请求验证再恢复；禁止仅改 `schedulable`。 +- P2：继续增强原生 system-log 对策略事件和最终 outcome 的索引，减少 runtime fallback 与 outcome missing。 + +## 验证 + +- `bun --check`：`runtime.ts`、`remote-python-sync.ts`、`output.ts` 通过。 +- `git diff --check` 通过。 +- `bun test scripts/src/output.test.ts`：2 pass，0 fail。 +- 默认 `runtime errors --all-groups`：紧凑表格，无 JSON envelope，无 `outputTruncated`。 +- 默认 `runtime errors --group 3`：紧凑详情、共享证据提示和 request 索引可见。 +- 默认 `runtime list`：Kubernetes 风格账号列表。 +- 显式 `--json`：返回机器可读结构。 +- degraded trace 与两条 failover-no-candidate trace 均通过真实 PK01 原生 API 回归。 + +## 后续 + +- 父任务 R2.2 保持进行中，继续按完整窗口观察调优效果。 +- 未创建外部 GitHub issue；相关未决项继续由本 MDTODO 问题域跟踪，避免未经授权写入外部系统。 diff --git a/docs/MDTODO/pr-merge-driven-automatic-delivery.md b/docs/MDTODO/pr-merge-driven-automatic-delivery.md index 161f014f..e1448d99 100644 --- a/docs/MDTODO/pr-merge-driven-automatic-delivery.md +++ b/docs/MDTODO/pr-merge-driven-automatic-delivery.md @@ -83,7 +83,7 @@ 合并修复 PR 后只观察新的 GitHub webhook→Gitea mirror→PaC→Tekton→GitOps/Argo 自动事件，验证 watcher Ready、TaskRun 创建及 AgentRun/Monitor runtime 收敛，不人工补齐既有 queued run，完成任务后将详细报告写入[任务报告](./details/pr-merge-driven-automatic-delivery/R5.3_Task_Report.md)。 -## R6 [in_progress] +## R6 [completed] 执行 [UniDesk #1981](https://github.com/pikasTech/unidesk/issues/1981)：固定仓库产品功能 schema 路径为 config/feature-config.schema.json，CI/CD 只读校验 schema 语法、值匹配和功能到唯一配置变量的一对一关系；缺失、非法、不匹配或重复仅输出 typed warning 并投影到 OTel/status/history/debug-step，必须继续 artifact、GitOps/Argo 和滚动上线，禁止写回、默认值、变量删除、功能关闭、架构切换及其他补救操作，完成任务后将详细报告写入[任务报告](./details/pr-merge-driven-automatic-delivery/R6_Task_Report.md)。 diff --git a/docs/MDTODO/sub2api-upstream-reliability.md b/docs/MDTODO/sub2api-upstream-reliability.md new file mode 100644 index 00000000..8d56aa2c --- /dev/null +++ b/docs/MDTODO/sub2api-upstream-reliability.md @@ -0,0 +1,37 @@ +# Sub2API 上游可靠性 + +本问题域持续跟踪 Sub2API 上游错误的真实归因、账号运行时可见性、受控调度缓解和用户侧错误隔离。配置事实以 owning YAML 为准，手工账号运行时配置通过受控 CRUD 显式管理。 + + +## R1 [completed] + +完善 Sub2API 上游错误归因与 runtime CRUD 可见性，并基于实际错误证据应用保守的临时不可调度策略，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R1_Task_Report.md)。 + +## R2 [in_progress] + +在新观测窗口复核临时不可调度命中、内部换号和最终客户响应，按请求分母评估调优效果并识别仍会穿透的流式错误，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2_Task_Report.md)。 + +### R2.1 [completed] + +增强 CLI 使用 Sub2API 原生 admin/ops 数据诊断请求错误、账号可用性、并发和 request trace，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.1_Task_Report.md)。 + +### R2.2 [in_progress] + +在连续新窗口复核临时不可调度、failover、客户可见错误率和流式 degraded 请求，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2_Task_Report.md)。 + +#### R2.2.1 [completed] + +溯源最近窗口客户可见错误请求，区分上游失败、切号耗尽、无候选和流式提交后错误并给出配置调优建议，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.1_Task_Report.md)。 +#### R2.2.2 [completed] + +执行客户错误调优：处理 gpt-5.4-nano 无候选，扩展 429 upstream rate limit 与精确 504 临时不可调度规则，并通过受控 runtime apply 下发和验证，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.2_Task_Report.md)。 +#### R2.2.3 [completed] + +将 runtime 精准 CRUD、批量操作、原生模型探测、客户可见错误溯源和基于真实证据的配置调优流程沉淀到 unidesk-sub2api skill，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.3_Task_Report.md)。 +#### R2.2.4 [completed] + +重新分析所有 Sub2API 分组最近的用户可见错误，并把 runtime errors 泛化为 all-groups 与精确 group 下钻；默认输出必须提供语义摘要、索引和渐进披露，禁止用手工 limit 规避输出问题，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.4_Task_Report.md)。 + +### R2.3 + +调查并收敛 Sub2API 原生 system-log 索引未保留策略事件的能力缺口，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.3_Task_Report.md)。 diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 53ac73f3..a4df1fda 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -157,8 +157,26 @@ - `runtime.templates` 保存可复用的 Sub2API runtime 规则模板。模板本身不是账号绑定，不会自动接管未在 YAML 中声明的手工账号。 - `pool.defaultTempUnschedulableTemplate` 是 YAML-managed 账号的默认临时不可调度模板引用；`profiles.entries[].tempUnschedulableTemplate` 可以为明确声明的账号选择其他模板。`codex-pool sync --confirm` 只向这些 managed 账号渲染 `temp_unschedulable_enabled` 和 `temp_unschedulable_rules`。 - YAML 未声明的账号属于 `runtime-manual`。`codex-pool runtime list|get` 可以显示其实际状态和 `matchingTemplate`，但与模板不同不构成 drift，也不得由 `sync` 自动纳管。 -- `codex-pool runtime errors` 从实际 Sub2API 日志按账号聚合错误。规则状态码只能取 `account_upstream_error` 的观测值；`openai.forward_failed` 只作为脱敏错误摘要证据，不能凭摘要推测未观测状态码。 +- `codex-pool runtime errors` 优先从 Sub2API 原生 admin/ops 能力读取服务数据： + - dashboard overview 提供请求分母、客户可见错误率和上游错误率； + - account availability 提供账号可用、限流、过载和错误状态； + - concurrency 提供组与账号容量、当前占用和排队； + - upstream errors 提供账号、状态码和根因详情； + - system-log 索引未保留策略事件时，才从 target runtime 日志补临时不可调度、failover、选号失败和 `forward_failed`，并披露 fallback； + - 规则状态码只能取 `account_upstream_error` 的观测值； + - `openai.forward_failed` 只作为脱敏错误摘要证据，不能凭摘要推测未观测状态码； + - Sub2API Ops 上游错误记录覆盖全部错误账号和状态码，默认输出全池与账号主因； + - `policyEffectiveness` 通过 `request_id` 关联临时不可调度、切号、无候选和最终网关响应，并提供可交给 `trace` 的 request id； + - 规则命中必须有 `account_temp_unschedulable` 日志证据，状态码相同不代表响应体关键词也命中； + - `requestIdCoverage` 必须披露事件关联缺口，缺少 request id 时不能用零关联证明策略链未发生； + - HTTP 200 与 `forward_failed` 可以在流式响应中并存，必须单列这类请求，不能仅凭最终 HTTP 状态判断客户收到完整终止事件； + - 窗口内最终错误占比只描述目标 API 路径的已完成日志事件，必须保留 `since`、`tail` 和分母覆盖边界； + - `--full` 才展开模型、小时、流式分布和有限详情样本； + - 账号当前并发只作为查询时快照，不能替代错误发生瞬间的历史并发证据； + - 没有账号全部路由尝试数时，只能报告错误数量，不能把错误记录数当作错误率； + - Ops 样本与 runtime 日志保持独立计数，不把时间邻近当作请求级关联。 - 手工账号的 runtime 配置必须通过 `codex-pool runtime apply|delete` 受控修改。写操作必须显式指定 `--account`，不带 `--confirm` 时只生成 dry-run，带 `--confirm` 才通过 Sub2API admin API 写入。 +- `codex-pool trace` 优先使用原生 `/admin/ops/requests` 和 `/admin/ops/system-logs`，target runtime 日志只补索引缺失的邻近事件；必须显式披露 target、runtime mode、主数据源、上下文数据源和读取状态。PK01 host-Docker 回退时不得固定读取 k3s Deployment，并且必须合并 Docker stdout/stderr 日志流。HTTP 200 与 `forward_failed` 并存且没有 failover 时应标记为 `degraded`。 - 内置临时不可调度配置与外部 `sentinel.*` 是独立控制面。runtime 模板负责请求路径即时冷却与 failover；`sentinel.*` 负责账号 marker health、隔离、恢复和 probe 周期，任一控制面都不得静默重写另一控制面的状态或规则。 - The external sentinel write surface is intentionally limited to the Sub2API admin `schedulable` action. Sentinel freeze/restore may set `schedulable=false|true`, but must not write, clear, or indirectly clear Sub2API request-path runtime state such as `temp_unschedulable_until`, `temp_unschedulable_reason`, rate-limit, overload, or model-rate-limit state. In particular, sentinel restore must not call Sub2API `recover-state`, because that endpoint is a broader runtime-state recovery operation rather than a pure schedulability restore. - Codex accounts selected by YAML do not declare `schedulable` as durable configuration. `codex-pool sync --confirm` must not restore existing account schedulability merely because YAML selects the account or sentinel state lacks an active quarantine. Existing `schedulable=false` is runtime state: the sentinel first reads Sub2API's actual account state, schedules a recovery probe for unschedulable managed accounts, and restores `schedulable=true` only after the marker probe matches. diff --git a/scripts/src/output.ts b/scripts/src/output.ts index 4c217e03..2cc8202c 100644 --- a/scripts/src/output.ts +++ b/scripts/src/output.ts @@ -307,7 +307,7 @@ function renderTextOutput(command: string, text: string): string { trigger, ...(dump === null ? {} : { dump }), next: [ - "Use rg --max-count, head, tail, --limit, --tail-bytes, or an id-specific query.", + "Use the command's semantic summary, page cursor, stable id, or id-specific detail query before requesting complete output.", "Use --full or --raw only when the complete one-off payload is intentionally required.", ], }, @@ -341,7 +341,7 @@ function oversizedOutputNext(envelope: JsonEnvelope): string[] { ]; } return [ - "Use --limit, --tail-bytes, an id-specific view, or another semantic narrow query.", + "Use the command's semantic summary, page cursor, stable id, or id-specific detail query; improve the command when none is available.", "Use --full or --raw only when the complete one-off payload is intentionally required.", ]; } diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts index 4facf494..88c75360 100644 --- a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts @@ -2867,6 +2867,11 @@ def classify_trace_event(item, account_names_by_id): "error": item.get("error"), "excludedAccountCount": item.get("excluded_account_count"), }) + elif "openai.forward_failed" in message: + event.update({ + "type": "forward-failed", + "error": item.get("error"), + }) elif "account_upstream_error" in message: event.update({ "type": "upstream-error", @@ -2939,10 +2944,15 @@ def trace_reason(events, final_event): failovers = [item for item in events if item.get("type") == "failover"] select_failures = [item for item in events if item.get("type") == "select-failed"] upstream_errors = [item for item in events if item.get("type") == "upstream-error"] + forward_failures = [item for item in events if item.get("type") == "forward-failed"] if failover_budget_exhausted(failovers, final_event): return "failover-budget-exhausted" if failovers and select_failures: return "failover-attempted-no-candidate" + if forward_failures and isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400: + return "forward-failed-http-success" + if forward_failures: + return "forward-failed" if failovers: return "failover-attempted" if select_failures: @@ -3006,6 +3016,60 @@ def run_trace(): if not isinstance(request_id, str) or not request_id: raise RuntimeError("trace payload missing requestId") admin_email, token, admin_compliance = login() + request_query = quote(request_id, safe="") + native_request_status = {"status": "unavailable"} + native_request = None + try: + request_data = ensure_success( + curl_api("GET", f"/api/v1/admin/ops/requests?request_id={request_query}&kind=all&page=1&page_size=100", bearer=token), + "get ops request details", + ) + request_items = extract_items(request_data) + native_request = next((item for item in request_items if isinstance(item, dict) and item.get("request_id") == request_id), None) + native_request_status = { + "status": "available", + "matched": native_request is not None, + "returned": len(request_items), + "total": request_data.get("total", len(request_items)) if isinstance(request_data, dict) else len(request_items), + } + except Exception as exc: + native_request_status = {"status": "unavailable", "reason": text(str(exc), 500)} + + native_system_log_status = {"status": "unavailable"} + native_matched = [] + try: + system_log_data = ensure_success( + curl_api("GET", f"/api/v1/admin/ops/system-logs?request_id={request_query}&page=1&page_size=200", bearer=token), + "get ops request system logs", + ) + system_log_items = extract_items(system_log_data) + for row in system_log_items: + if not isinstance(row, dict): + continue + parsed = dict(row.get("extra") or {}) if isinstance(row.get("extra"), dict) else {} + for source_key in ("request_id", "client_request_id", "account_id", "platform", "model"): + if parsed.get(source_key) is None and row.get(source_key) is not None: + parsed[source_key] = row.get(source_key) + parsed["_at"] = row.get("created_at") + parsed["_message"] = row.get("message") + parsed["_line"] = json.dumps({ + "created_at": row.get("created_at"), + "message": row.get("message"), + "request_id": row.get("request_id"), + "account_id": row.get("account_id"), + "platform": row.get("platform"), + "model": row.get("model"), + "extra": parsed, + }, ensure_ascii=False, default=str) + native_matched.append(parsed) + native_system_log_status = { + "status": "available", + "returned": len(system_log_items), + "total": system_log_data.get("total", len(system_log_items)) if isinstance(system_log_data, dict) else len(system_log_items), + } + except Exception as exc: + native_system_log_status = {"status": "unavailable", "reason": text(str(exc), 500)} + account_snapshot, account_snapshot_error = account_snapshot_from_runtime(token) account_names_by_id = {} for row in account_snapshot: @@ -3014,17 +3078,21 @@ def run_trace(): account_id = int(account_id) if isinstance(account_id, int) and isinstance(row.get("accountName"), str): account_names_by_id[account_id] = row.get("accountName") - proc = kubectl(["-n", NAMESPACE, "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"]) - stdout = proc.stdout.decode("utf-8", errors="replace") + proc = runtime_logs(since, tail) + log_bytes = proc.stdout + b"\\n" + proc.stderr + stdout = log_bytes.decode("utf-8", errors="replace") parsed_lines = [] - matched = [] + runtime_matched = [] for line in stdout.splitlines(): parsed = parse_log_line(line) if parsed is None: continue parsed_lines.append(parsed) if request_id in line: - matched.append(parsed) + runtime_matched.append(parsed) + matched = native_matched if native_matched else runtime_matched + matched.sort(key=lambda item: (log_time_epoch(item) is None, log_time_epoch(item) or 0)) + trace_event_source = "sub2api-native-admin-ops-system-logs" if native_matched else "runtime-logs-fallback" first_epoch = None last_epoch = None for item in matched: @@ -3045,22 +3113,55 @@ def run_trace(): window_lines = matched events = [classify_trace_event(item, account_names_by_id) for item in matched] request_start = next((item for item in events if item.get("type") == "request-start"), None) + if isinstance(native_request, dict): + if not isinstance(request_start, dict): + request_start = { + "type": "request-start", + "at": native_request.get("created_at"), + "requestId": native_request.get("request_id"), + "path": None, + "model": native_request.get("model"), + "accountId": native_request.get("account_id"), + "accountName": account_names_by_id.get(native_request.get("account_id")), + "stream": native_request.get("stream"), + "source": "sub2api-native-admin-ops-requests", + } + else: + if request_start.get("stream") is None: + request_start["stream"] = native_request.get("stream") + if not request_start.get("model"): + request_start["model"] = native_request.get("model") final_event = next((item for item in reversed(events) if item.get("type") == "final"), None) + if not isinstance(final_event, dict) and isinstance(native_request, dict) and isinstance(native_request.get("status_code"), int): + final_event = { + "type": "final", + "at": native_request.get("created_at"), + "requestId": native_request.get("request_id"), + "path": None, + "model": native_request.get("model"), + "accountId": native_request.get("account_id"), + "accountName": account_names_by_id.get(native_request.get("account_id")), + "statusCode": native_request.get("status_code"), + "latencyMs": native_request.get("duration_ms"), + "source": "sub2api-native-admin-ops-requests", + } failovers = [item for item in events if item.get("type") == "failover"] select_failures = [item for item in events if item.get("type") == "select-failed"] upstream_errors = [item for item in events if item.get("type") == "upstream-error"] + forward_failures = [item for item in events if item.get("type") == "forward-failed"] temp_unsched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if "account_temp_unschedulable" in str(item.get("_message") or "")] admin_sched = [with_trace_phase(classify_trace_event(item, account_names_by_id), first_epoch, last_epoch) for item in window_lines if ("schedulable" in str(item.get("_message") or "") or "/schedulable" in str(item.get("path") or ""))] window_events = [classify_trace_event(item, account_names_by_id) for item in window_lines] final_errors = [item for item in window_events if item.get("type") == "final" and isinstance(item.get("statusCode"), int) and item.get("statusCode") >= 400] window_failovers = [item for item in window_events if item.get("type") == "failover"] window_select_failures = [item for item in window_events if item.get("type") == "select-failed"] + window_forward_failures = [item for item in window_events if item.get("type") == "forward-failed"] untried_schedulable_accounts = trace_untried_schedulable_accounts(failovers, final_event or {}, account_snapshot) reason = trace_reason(events, final_event) if not matched: outcome = "not-found" elif isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400: - outcome = "succeeded" + outcome = "degraded" if forward_failures else ("succeeded-with-failover" if failovers else "succeeded") elif isinstance(final_event, dict): outcome = "failed" else: @@ -3068,11 +3169,14 @@ def run_trace(): return { "ok": proc.returncode == 0 and len(matched) > 0, "mode": "trace", + "targetId": TARGET_ID, + "runtimeMode": RUNTIME_MODE, "namespace": NAMESPACE, "serviceDns": SERVICE_DNS, "appPod": APP_POD, "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, "requestId": request_id, + "eventSource": trace_event_source, "summary": { "outcome": outcome, "reason": reason, @@ -3090,10 +3194,16 @@ def run_trace(): }, "request": request_start or {}, "final": final_event or {}, + "nativeOps": { + "requestDetails": native_request_status, + "request": native_request, + "systemLogs": native_system_log_status, + }, "events": events, "failovers": failovers, "selectFailures": select_failures, "upstreamErrors": upstream_errors, + "forwardFailures": forward_failures, "tempUnschedulable": temp_unsched, "adminSchedulable": admin_sched[-20:], "windowStats": { @@ -3102,6 +3212,8 @@ def run_trace(): "finalErrorCount": len(final_errors), "failoverCount": len(window_failovers), "selectFailedCount": len(window_select_failures), + "forwardFailedCount": len(window_forward_failures), + "matchedForwardFailedCount": len(forward_failures), "tempUnschedulableCount": len(temp_unsched), "adminSchedulableCount": len(admin_sched), }, @@ -3114,8 +3226,10 @@ def run_trace(): "rawLines": [{"line": item.get("_line")} for item in matched[-30:]] if show_lines else [], "showLines": show_lines, "logs": { + "source": trace_event_source, + "contextSource": f"host-docker:{HOST_DOCKER_APP_CONTAINER}" if RUNTIME_MODE == "host-docker" else f"k3s:{NAMESPACE}/deployment/sub2api", "exitCode": proc.returncode, - "stderrTail": text(proc.stderr, 1000), + "stderrTail": text(proc.stderr, 1000) if proc.returncode != 0 else "", "stdoutLineCount": len(stdout.splitlines()), }, "valuesPrinted": False, diff --git a/scripts/src/platform-infra-sub2api-codex/render.ts b/scripts/src/platform-infra-sub2api-codex/render.ts index a8898fc0..a7052e38 100644 --- a/scripts/src/platform-infra-sub2api-codex/render.ts +++ b/scripts/src/platform-infra-sub2api-codex/render.ts @@ -398,8 +398,10 @@ export function renderTraceReport( const failovers = recordArray(parsed.failovers); const selectFailures = recordArray(parsed.selectFailures); const upstreamErrors = recordArray(parsed.upstreamErrors); + const forwardFailures = recordArray(parsed.forwardFailures); const tempUnschedulable = recordArray(parsed.tempUnschedulable); const windowStats = isRecord(parsed.windowStats) ? parsed.windowStats : {}; + const logs = isRecord(parsed.logs) ? parsed.logs : {}; const accountSnapshot = recordArray(parsed.accountSnapshot); const lines: string[] = []; lines.push([ @@ -409,6 +411,17 @@ export function renderTraceReport( `outcome=${stringValue(summary.outcome) ?? "-"}`, `reason=${stringValue(summary.reason) ?? "-"}`, ].join(" ")); + lines.push([ + "SOURCE", + `target=${textValue(parsed.targetId)}`, + `runtime=${textValue(parsed.runtimeMode)}`, + `logs=${textValue(logs.source)}`, + `context=${textValue(logs.contextSource)}`, + `exit=${textValue(logs.exitCode)}`, + ].join(" ")); + if (parsed.ok !== true && stringValue(logs.stderrTail) !== null) { + lines.push(`LOG ERROR ${shorten(stringValue(logs.stderrTail) ?? "", 240)}`); + } lines.push([ "REQUEST", `path=${request.path ?? final.path ?? "-"}`, @@ -452,6 +465,19 @@ export function renderTraceReport( ]), ])); } + if (forwardFailures.length > 0) { + lines.push(""); + lines.push("FORWARD-FAILED"); + lines.push(renderTable([ + ["AT", "ACCOUNT", "STATUS", "ERROR"], + ...forwardFailures.map((item) => [ + shortIso(item.at), + formatAccountRef(item), + textValue(item.statusCode), + shorten(stringValue(item.error) ?? "-", 72), + ]), + ])); + } if (upstreamErrors.length > 0 || tempUnschedulable.length > 0) { lines.push(""); lines.push("ACCOUNT SIGNALS"); @@ -482,13 +508,14 @@ export function renderTraceReport( lines.push(""); lines.push("WINDOW STATS"); lines.push(renderTable([ - ["MATCH", "EVENTS", "FINAL_4XX_5XX", "FAILOVER", "SELECT_FAIL", "TEMP_UNSCHED", "ADMIN_SCHED"], + ["MATCH", "EVENTS", "FINAL_4XX_5XX", "FAILOVER", "SELECT_FAIL", "FORWARD_FAIL", "TEMP_UNSCHED", "ADMIN_SCHED"], [ textValue(windowStats.matchedLines), textValue(windowStats.eventCount), textValue(windowStats.finalErrorCount), textValue(windowStats.failoverCount), textValue(windowStats.selectFailedCount), + textValue(windowStats.forwardFailedCount), textValue(windowStats.tempUnschedulableCount), textValue(windowStats.adminSchedulableCount), ], diff --git a/scripts/src/platform-infra-sub2api-codex/runtime.ts b/scripts/src/platform-infra-sub2api-codex/runtime.ts index e5f1b561..103ca8b0 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime.ts @@ -1,10 +1,12 @@ import type { UniDeskConfig } from "../config"; +import type { RenderedCliResult } from "../output"; import { desiredAccountNames } from "./accounts"; import { readCodexPoolConfig } from "./config"; import { protectedManualAccountsForTarget } from "./public-exposure"; import { renderSub2ApiTempUnschedulableCredentials } from "./redaction"; import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote"; +import { renderedCliResult, renderTable } from "./render"; import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target"; type RuntimeAction = "list" | "get" | "errors" | "apply" | "delete"; @@ -12,17 +14,22 @@ type RuntimeAction = "list" | "get" | "errors" | "apply" | "delete"; interface RuntimeOptions { action: RuntimeAction; account: string | null; + group: string | null; + allGroups: boolean; + platform: string | null; + pageToken: string | null; template: string | null; kind: "temp-unschedulable"; confirm: boolean; full: boolean; raw: boolean; + json: boolean; since: string; tail: number; targetId: string; } -export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise> { +export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { const options = parseRuntimeOptions(args); const pool = readCodexPoolConfig(); const target = codexPoolRuntimeTarget(options.targetId); @@ -33,6 +40,10 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P const payload = { action: options.action, account: options.account, + group: options.group, + allGroups: options.allGroups, + platform: options.platform, + pageToken: options.pageToken, kind: options.kind, confirm: options.confirm, full: options.full, @@ -61,12 +72,12 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P return { ok, action: "platform-infra-sub2api-codex-pool-runtime", - remote: compactCapture(result, { full: true }), + remote: compactCapture(result, { full: parsed === null || result.exitCode !== 0 }), parsed, valuesPrinted: false, }; } - return { + const response = { ok, action: "platform-infra-sub2api-codex-pool-runtime", target: { @@ -79,14 +90,18 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P request: { operation: options.action, account: options.account, + group: options.group, + allGroups: options.allGroups, + platform: options.platform, + pageToken: options.pageToken, template: options.template, kind: options.kind, confirm: options.confirm, since: options.action === "errors" ? options.since : undefined, tail: options.action === "errors" ? options.tail : undefined, }, - runtime: parsed, - remote: compactCapture(result, { full: options.full || result.exitCode !== 0 || parsed === null }), + runtime: compactRuntimeErrors(parsed, options), + remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }), disclosure: options.action === "list" ? { get: "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account ", @@ -95,19 +110,327 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P : undefined, valuesPrinted: false, }; + return options.full || options.json ? response : renderRuntimeResult(response, options); +} + +function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown { + if (options.action !== "errors" || options.full || value === null || typeof value !== "object" || Array.isArray(value)) return value; + const runtime = value as Record; + const allGroups = runtimeRecord(runtime.allGroups); + if (allGroups !== null) { + const nextPageToken = typeof allGroups.nextPageToken === "string" ? allGroups.nextPageToken : null; + const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; + const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; + return { + ok: runtime.ok, + operation: runtime.operation, + mutation: runtime.mutation, + endpoint: runtime.endpoint, + allGroups, + disclosure: { + group: `${baseCommand} --group ${platformFlag}`, + nextPage: nextPageToken === null ? null : `${baseCommand} --all-groups${platformFlag} --page-token ${nextPageToken}`, + note: "The overview uses stable group-id pagination. Use group drilldown for account root causes and request indexes.", + }, + valuesPrinted: false, + }; + } + const errors = runtimeRecord(runtime.errors); + if (errors === null) return value; + const effectiveness = runtimeRecord(errors.policyEffectiveness) ?? {}; + const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; + const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; + const availabilityGroup = runtimeRecord(availability.group) ?? {}; + const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; + const concurrencyGroup = runtimeRecord(concurrency.group) ?? {}; + const sink = runtimeRecord(nativeOps.systemLogSink) ?? {}; + const sourceStatus = runtimeRecord(effectiveness.sourceStatus) ?? {}; + const drilldown = runtimeRecord(effectiveness.requestDrilldown) ?? {}; + const requestIndexes = Object.fromEntries(Object.entries(drilldown).flatMap(([key, item]) => { + if (!Array.isArray(item)) return []; + const ids = item.filter((entry): entry is string => typeof entry === "string"); + if (ids.length === 0) return []; + return [[key, { + count: ids.length, + sampleRequestId: ids[0], + moreAvailable: ids.length > 1, + }]]; + })); + const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; + const groupFlag = options.group === null ? "" : ` --group ${options.group}`; + const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; + return { + ok: runtime.ok, + operation: runtime.operation, + mutation: runtime.mutation, + group: runtime.group, + endpoint: runtime.endpoint, + errors: { + window: errors.window, + nativeOps: { + source: nativeOps.source, + overview: nativeOps.overview, + accountAvailability: { + status: availability.status, + totalAccounts: availabilityGroup.total_accounts, + availableCount: availabilityGroup.available_count, + rateLimitCount: availabilityGroup.rate_limit_count, + errorCount: availabilityGroup.error_count, + unavailableAccounts: Array.isArray(availability.unavailableAccounts) + ? availability.unavailableAccounts.map((item) => { + const account = runtimeRecord(item) ?? {}; + return { + accountId: account.accountId, + accountName: account.accountName, + status: account.status, + reason: account.error, + }; + }) + : [], + }, + concurrency: { + status: concurrency.status, + currentInUse: concurrencyGroup.current_in_use, + maxCapacity: concurrencyGroup.max_capacity, + loadPercentage: concurrencyGroup.load_percentage, + waitingInQueue: concurrencyGroup.waiting_in_queue, + }, + systemLogSink: { + status: sink.status, + queueDepth: sink.queueDepth, + droppedCount: sink.droppedCount, + writeFailedCount: sink.writeFailedCount, + }, + }, + rootCauseAnalysis: errors.rootCauseAnalysis, + policyEffectiveness: { + source: effectiveness.source, + scope: effectiveness.scope, + attribution: effectiveness.attribution, + assessment: effectiveness.assessment, + ruleMatches: effectiveness.ruleMatches, + failover: effectiveness.failover, + forwardFailure: effectiveness.forwardFailure, + selectFailedEvents: effectiveness.selectFailedEvents, + requestIdCoverage: effectiveness.requestIdCoverage, + gatewayOutcomeEvidence: effectiveness.gatewayOutcomeEvidence, + requestIndexes, + sourceStatus: { + status: sourceStatus.status, + source: sourceStatus.source, + fallbackUsed: sourceStatus.fallbackUsed, + fallbackReason: sourceStatus.fallbackReason, + fallbackEventCount: sourceStatus.fallbackEventCount, + }, + }, + }, + disclosure: { + complete: `${baseCommand}${groupFlag}${platformFlag}${options.account === null ? "" : " --account "} --full`, + account: `${baseCommand}${groupFlag}${platformFlag} --account `, + trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", + note: "Use requestIndexes for trace drilldown and --full for the complete finite index.", + }, + valuesPrinted: false, + }; +} + +function runtimeRecord(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : null; +} + +function runtimeRecords(value: unknown): Record[] { + return Array.isArray(value) ? value.flatMap((item) => { + const record = runtimeRecord(item); + return record === null ? [] : [record]; + }) : []; +} + +function runtimeText(value: unknown): string { + if (value === null || value === undefined || value === "") return "-"; + if (typeof value === "boolean") return value ? "true" : "false"; + if (Array.isArray(value)) return value.map(runtimeText).join(","); + return String(value).replace(/\s+/gu, " ").trim(); +} + +function runtimeShort(value: unknown, max = 42): string { + const text = runtimeText(value); + return text.length <= max ? text : `${text.slice(0, max - 1)}…`; +} + +function runtimeRate(value: unknown): string { + return typeof value === "number" && Number.isFinite(value) ? `${(value * 100).toFixed(2)}%` : "-"; +} + +function renderRuntimeResult(response: Record, options: RuntimeOptions): RenderedCliResult { + const runtime = runtimeRecord(response.runtime) ?? {}; + const ok = response.ok === true; + const lines: string[] = []; + if (options.action === "errors") renderRuntimeErrors(lines, runtime, options); + else if (options.action === "list") renderRuntimeAccountList(lines, runtime); + else if (options.action === "get") renderRuntimeAccountDetail(lines, runtime); + else renderRuntimeMutation(lines, runtime, options); + lines.push("", `JSON: bun scripts/cli.ts platform-infra sub2api codex-pool runtime ${options.action} --json`); + return renderedCliResult(ok, `platform-infra sub2api codex-pool runtime ${options.action}`, lines.join("\n")); +} + +function renderRuntimeErrors(lines: string[], runtime: Record, options: RuntimeOptions): void { + const allGroups = runtimeRecord(runtime.allGroups); + if (allGroups !== null) { + const totals = runtimeRecord(allGroups.pageTotals) ?? {}; + const groups = runtimeRecords(allGroups.groups); + lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=all-groups since=${options.since}`); + lines.push(renderTable([ + ["GROUPS", "RETURNED", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UPSTREAM_ERR", "UP_RATE", "ATTENTION"], + [ + runtimeText(allGroups.totalGroups), runtimeText(allGroups.returnedGroups), runtimeText(totals.requestCount), + runtimeText(totals.errorCount), runtimeRate(totals.errorRate), runtimeText(totals.upstreamErrorCount), + runtimeRate(totals.upstreamErrorRate), runtimeText(totals.groupsNeedingAttention), + ], + ])); + lines.push("", "GROUPS"); + lines.push(renderTable([ + ["NAME", "ID", "PLATFORM", "STATUS", "REQ", "CLIENT_ERR", "ERR_RATE", "UP_ERR", "UP_RATE", "ACCOUNTS", "AVAILABLE", "USE/CAP", "QUEUE", "ATTN"], + ...groups.map((group) => [ + runtimeShort(group.groupName, 28), runtimeText(group.groupId), runtimeText(group.platform), runtimeText(group.status), + runtimeText(group.requestCount), runtimeText(group.errorCount), runtimeRate(group.errorRate), runtimeText(group.upstreamErrorCount), + runtimeRate(group.upstreamErrorRate), runtimeText(group.totalAccounts), runtimeText(group.availableCount), + `${runtimeText(group.currentInUse)}/${runtimeText(group.maxCapacity)}`, runtimeText(group.waitingInQueue), group.needsAttention === true ? "yes" : "no", + ]), + ])); + const disclosure = runtimeRecord(runtime.disclosure) ?? {}; + lines.push("", `Detail: ${runtimeText(disclosure.group)}`); + if (typeof disclosure.nextPage === "string") lines.push(`Next: ${disclosure.nextPage}`); + return; + } + const group = runtimeRecord(runtime.group) ?? {}; + const errors = runtimeRecord(runtime.errors) ?? {}; + const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; + const overview = runtimeRecord(nativeOps.overview) ?? {}; + const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; + const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; + lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`); + lines.push(renderTable([ + ["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"], + [ + runtimeShort(group.name, 30), runtimeText(group.id), runtimeText(group.platform), runtimeText(overview.requestCount), + runtimeText(overview.errorCount), runtimeRate(overview.errorRate), runtimeRate(overview.upstreamErrorRate), + runtimeText(overview.businessLimitedCount), runtimeText(overview.healthScore), + ], + ])); + lines.push(renderTable([ + ["ACCOUNTS", "AVAILABLE", "RATE_LIMIT", "ERROR", "IN_USE", "CAPACITY", "QUEUE"], + [ + runtimeText(availability.totalAccounts), runtimeText(availability.availableCount), runtimeText(availability.rateLimitCount), + runtimeText(availability.errorCount), runtimeText(concurrency.currentInUse), runtimeText(concurrency.maxCapacity), runtimeText(concurrency.waitingInQueue), + ], + ])); + const unavailable = runtimeRecords(availability.unavailableAccounts); + if (unavailable.length > 0) { + lines.push("", "UNAVAILABLE ACCOUNTS", renderTable([ + ["ACCOUNT", "ID", "STATUS", "REASON"], + ...unavailable.map((account) => [runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.status), runtimeShort(account.reason, 70)]), + ])); + } + const root = runtimeRecord(errors.rootCauseAnalysis) ?? {}; + const causes = runtimeRecords(root.causeBuckets); + if (causes.length > 0) { + lines.push("", `ROOT CAUSES scope=${runtimeText(root.scope)}`, renderTable([ + ["CAUSE", "COUNT"], + ...causes.map((cause) => [runtimeText(cause.cause), runtimeText(cause.count)]), + ])); + } + const policy = runtimeRecord(errors.policyEffectiveness) ?? {}; + const rules = runtimeRecord(policy.ruleMatches) ?? {}; + const failover = runtimeRecord(policy.failover) ?? {}; + const forward = runtimeRecord(policy.forwardFailure) ?? {}; + lines.push("", `POLICY EFFECTIVENESS scope=${runtimeText(policy.scope)}`); + lines.push(renderTable([ + ["TEMP_UNSCHED", "FAILOVER", "SUCCESS", "FAILED", "MISSING", "FORWARD_200", "FORWARD_ERR", "SELECT_FAIL"], + [ + runtimeText(rules.tempUnschedulableEvents), runtimeText(failover.requests), runtimeText(failover.succeeded), + runtimeText(failover.failed), runtimeText(failover.outcomeMissing), runtimeText(forward.withHttpSuccess), + runtimeText(forward.withHttpError), runtimeText(policy.selectFailedEvents), + ], + ])); + const indexes = runtimeRecord(policy.requestIndexes) ?? {}; + const indexRows = Object.entries(indexes).flatMap(([kind, value]) => { + const item = runtimeRecord(value); + return item === null ? [] : [[kind, runtimeText(item.count), runtimeText(item.sampleRequestId), item.moreAvailable === true ? "yes" : "no"]]; + }); + if (indexRows.length > 0) lines.push("", "REQUEST INDEX", renderTable([["TYPE", "COUNT", "SAMPLE_REQUEST_ID", "MORE"], ...indexRows])); + if (typeof root.attribution === "string") lines.push("", `NOTE: ${root.attribution}`); + if (typeof policy.attribution === "string") lines.push(`NOTE: ${policy.attribution}`); + const disclosure = runtimeRecord(runtime.disclosure) ?? {}; + lines.push("", `Account: ${runtimeText(disclosure.account)}`, `Trace: ${runtimeText(disclosure.trace)}`, `Full: ${runtimeText(disclosure.complete)}`); +} + +function renderRuntimeAccountList(lines: string[], runtime: Record): void { + const group = runtimeRecord(runtime.group) ?? {}; + const accounts = runtimeRecords(runtime.accounts); + lines.push(`SUB2API RUNTIME ACCOUNTS ok=${runtime.ok === true ? "true" : "false"} group=${runtimeText(group.name)} count=${accounts.length}`); + lines.push(renderTable([ + ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "CONCURRENCY", "LOAD", "PRIORITY", "TEMP_RULES", "TEMPLATE"], + ...accounts.map((account) => { + const temp = runtimeRecord(account.tempUnschedulable) ?? {}; + return [ + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), + runtimeText(account.schedulable), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, + runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(temp.ruleCount), runtimeText(account.matchingTemplate), + ]; + }), + ])); + lines.push("", "Detail: bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account "); +} + +function renderRuntimeAccountDetail(lines: string[], runtime: Record): void { + const account = runtimeRecord(runtime.account) ?? {}; + const temp = runtimeRecord(account.tempUnschedulable) ?? {}; + lines.push(`SUB2API RUNTIME ACCOUNT ok=${runtime.ok === true ? "true" : "false"}`); + lines.push(renderTable([ + ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "PROXY", "CONCURRENCY", "LOAD", "PRIORITY", "TEMPLATE"], + [ + runtimeShort(account.accountName, 42), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), + runtimeText(account.schedulable), runtimeText(account.proxyId), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, + runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(account.matchingTemplate), + ], + ])); + lines.push(renderTable([ + ["TEMP_ENABLED", "RULES", "STATUS_CODES", "POOL_MODE"], + [runtimeText(temp.enabled), runtimeText(temp.ruleCount), runtimeText(temp.statusCodes), runtimeText(account.poolModeEnabled)], + ])); +} + +function renderRuntimeMutation(lines: string[], runtime: Record, options: RuntimeOptions): void { + const plan = runtimeRecord(runtime.plan) ?? {}; + const account = runtimeRecord(plan.account) ?? runtimeRecord(runtime.account) ?? {}; + const before = runtimeRecord(plan.before) ?? {}; + const after = runtimeRecord(plan.after) ?? {}; + lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); + lines.push(renderTable([ + ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES"], + [ + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), + runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), + ], + ])); } export function parseRuntimeOptions(args: string[]): RuntimeOptions { const [actionRaw = "list", ...rest] = args; if (actionRaw !== "list" && actionRaw !== "get" && actionRaw !== "errors" && actionRaw !== "apply" && actionRaw !== "delete") { - throw new Error("runtime usage: list|get|errors|apply|delete [--account ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--full|--raw]"); + throw new Error("runtime usage: list|get|errors|apply|delete [--account ] [--group |--all-groups] [--platform ] [--page-token ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--json|--full|--raw]"); } let account: string | null = null; + let group: string | null = null; + let allGroups = false; + let platform: string | null = null; + let pageToken: string | null = null; let template: string | null = null; let kind = "temp-unschedulable" as const; let confirm = false; let full = false; let raw = false; + let json = false; let since = "24h"; let tail = 50_000; let targetId = defaultCodexPoolRuntimeTargetId(); @@ -122,8 +445,17 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { if (arg === "--confirm") confirm = true; else if (arg === "--full") full = true; else if (arg === "--raw") raw = true; + else if (arg === "--json") json = true; + else if (arg === "-o" && readValue("-o") === "json") json = true; else if (arg === "--account") account = readValue("--account"); else if (arg.startsWith("--account=")) account = arg.slice("--account=".length).trim(); + else if (arg === "--group") group = readValue("--group"); + else if (arg.startsWith("--group=")) group = arg.slice("--group=".length).trim(); + else if (arg === "--all-groups") allGroups = true; + else if (arg === "--platform") platform = readValue("--platform"); + else if (arg.startsWith("--platform=")) platform = arg.slice("--platform=".length).trim(); + else if (arg === "--page-token") pageToken = readValue("--page-token"); + else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim(); else if (arg === "--template") template = readValue("--template"); else if (arg.startsWith("--template=")) template = arg.slice("--template=".length).trim(); else if (arg === "--kind") kind = parseRuntimeKind(readValue("--kind")); @@ -140,13 +472,20 @@ export function parseRuntimeOptions(args: string[]): RuntimeOptions { throw new Error(`runtime ${actionRaw} requires --account `); } if (account !== null && (account.length > 256 || /[\r\n]/u.test(account))) throw new Error("--account has an unsupported format"); + if (group !== null && (group.length > 256 || /[\r\n]/u.test(group))) throw new Error("--group has an unsupported format"); + if (platform !== null && (!/^[A-Za-z0-9._-]+$/u.test(platform) || platform.length > 64)) throw new Error("--platform has an unsupported format"); + if (pageToken !== null && (!/^\d+$/u.test(pageToken) || pageToken.length > 20)) throw new Error("--page-token must be a numeric group cursor"); + if (group !== null && allGroups) throw new Error("use only one of --group or --all-groups"); + if ((group !== null || allGroups || platform !== null || pageToken !== null) && actionRaw !== "errors") throw new Error(`runtime ${actionRaw} does not accept group scope options`); + if (pageToken !== null && !allGroups) throw new Error("--page-token requires --all-groups"); + if (account !== null && allGroups) throw new Error("--account cannot be combined with --all-groups; use --group first"); if (actionRaw === "apply" && !template) throw new Error("runtime apply requires --template "); if (actionRaw !== "apply" && template !== null) throw new Error(`runtime ${actionRaw} does not accept --template`); if (actionRaw !== "errors" && (since !== "24h" || tail !== 50_000)) throw new Error(`runtime ${actionRaw} does not accept --since or --tail`); if (!/^\d+[mhd]$/u.test(since)) throw new Error("--since must use m, h, or d"); if ((actionRaw === "list" || actionRaw === "get" || actionRaw === "errors") && confirm) throw new Error(`runtime ${actionRaw} does not accept --confirm`); - if (full && raw) throw new Error("use only one of --full or --raw"); - return { action: actionRaw, account, template, kind, confirm, full, raw, since, tail, targetId }; + if ([full, raw, json].filter(Boolean).length > 1) throw new Error("use only one of --json, --full, or --raw"); + return { action: actionRaw, account, group, allGroups, platform, pageToken, template, kind, confirm, full, raw, json, since, tail, targetId }; } function parseRuntimeKind(value: string): "temp-unschedulable" { @@ -172,6 +511,7 @@ import json import re import subprocess import sys +from datetime import datetime, timedelta, timezone from urllib.parse import quote RUNTIME_MODE = ${pyJson(target.runtimeMode)} @@ -324,11 +664,14 @@ def login(): raise RuntimeError("admin login response has no token") return token -def list_groups(token): - return items_of(data_of(curl_api("GET", "/api/v1/admin/groups/all?platform=openai", bearer=token), "list groups")) +def list_groups(token, platform=None): + suffix = "?platform=" + quote(str(platform), safe="") if platform else "" + return items_of(data_of(curl_api("GET", "/api/v1/admin/groups/all" + suffix, bearer=token), "list groups")) -def list_group_accounts(token, group_id): - path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500&platform=openai" +def list_group_accounts(token, group_id, platform=None): + path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500" + if platform: + path += "&platform=" + quote(str(platform), safe="") return items_of(data_of(curl_api("GET", path, bearer=token), "list group accounts")) def account_detail(token, account_id): @@ -372,6 +715,10 @@ def account_summary(detail, include_rules=False): "status": detail.get("status"), "schedulable": detail.get("schedulable"), "proxyId": detail.get("proxy_id"), + "concurrencyLimit": detail.get("concurrency"), + "currentConcurrency": detail.get("current_concurrency"), + "loadFactor": detail.get("load_factor"), + "priority": detail.get("priority"), "tempUnschedulable": {key: policy[key] for key in ("enabled", "ruleCount", "statusCodes")}, "matchingTemplate": matching_template(credentials), "poolModeEnabled": credentials.get("pool_mode") is True, @@ -412,15 +759,676 @@ def compact_error(value): compact = re.sub(r"(?i)\\b(api[_ -]?key|token)[=:]\\s*[^\\s,;]+", r"\\1=[REDACTED]", compact) return compact[:240] -def observed_runtime_errors(accounts): +def since_start_time(value): + match = re.fullmatch(r"(\\d+)([mhd])", str(value or "")) + if not match: + raise RuntimeError("unsupported runtime error window: " + str(value)) + amount = int(match.group(1)) + unit = match.group(2) + duration = timedelta(minutes=amount) if unit == "m" else timedelta(hours=amount) if unit == "h" else timedelta(days=amount) + return (datetime.now(timezone.utc) - duration).isoformat().replace("+00:00", "Z") + +def ops_error_cause(value): + lower = value.lower() + markers = { + "concurrency": ("concurrency limit", "concurrent request", "too many concurrent", "maximum concurrency", "max concurrency"), + "overload": ("overloaded", "server busy", "temporarily unavailable", "capacity exhausted"), + "model-route-unavailable": ("no available channel for model", "unsupported model", "model not found", "model_not_found"), + "context-window": ("context window", "context length", "maximum context"), + "continuation-payload": ("encrypted_content", "missing_required_parameter", "missing required parameter"), + "invalid-request": ("invalid_request", "invalid request"), + "rate-limit": ("rate limit", "too many requests"), + "access-denied": ("forbidden", "access denied", "permission denied"), + "not-found": ("not found", "does not exist"), + "incomplete-stream": ("ended before a terminal event", "missing terminal event", "before response.completed"), + "transport": ("do request failed", "unexpected eof", "connection reset", "stream read error", "stream closed", "internal_error; received from peer"), + "upstream-api-key-validation": ("failed to validate api key", "api key validation failed"), + "authentication": ("unauthorized", "invalid token", "token revoked", "invalid_grant"), + "generic-upstream-failure": ("upstream request failed", "upstream gateway error"), + } + for cause, values in markers.items(): + if any(marker in lower for marker in values): + return cause + if re.match(r"recovered upstream error \\d+\\b", lower): + return "status-only-upstream-failure" + return "unknown" + +def ops_error_analysis(token, account, log_statuses): + account_id = account.get("id") + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + path = f"/api/v1/admin/ops/upstream-errors?account_id={account_id}&view=all&page=1&page_size=500&start_time={start_time}" + response = curl_api("GET", path, bearer=token) + try: + data = data_of(response, "list account ops errors") + except Exception as exc: + return { + "source": "ops-upstream-errors", + "status": "unavailable", + "logObservedStatusCodes": log_statuses, + "reason": compact_error(str(exc)), + "assessment": "insufficient-evidence", + } + rows = items_of(data) + total = data.get("total", len(rows)) if isinstance(data, dict) else len(rows) + detail_limit = 24 if PAYLOAD.get("full") is True else 8 + analyzed = [] + for row in rows: + message = compact_error(row.get("message") if isinstance(row, dict) else "") + analyzed.append({"row": row, "evidence": message, "cause": ops_error_cause(message)}) + for entry in analyzed[:detail_limit]: + row = entry["row"] + error_id = row.get("id") if isinstance(row, dict) else None + if not isinstance(error_id, int): + continue + detail_response = curl_api("GET", f"/api/v1/admin/ops/upstream-errors/{error_id}", bearer=token) + try: + detail = data_of(detail_response, "get account 500 ops error") + except Exception: + continue + fields = [] + for key in ("message", "upstream_error_message", "upstream_error_detail", "error_body", "upstream_errors"): + value = detail.get(key) if isinstance(detail, dict) else None + if isinstance(value, str) and value.strip(): + fields.append(value) + evidence = compact_error(" ".join(fields)) + entry["evidence"] = evidence + entry["cause"] = ops_error_cause(evidence) + + causes = {} + statuses = {} + messages = {} + models = {} + hours = {} + stream_modes = {"stream": 0, "synchronous": 0, "unknown": 0} + concurrency_count = 0 + for entry in analyzed: + row = entry["row"] + error_id = row.get("id") if isinstance(row, dict) else None + evidence = entry["evidence"] + cause = entry["cause"] + causes[cause] = causes.get(cause, 0) + 1 + list_message = compact_error(row.get("message") if isinstance(row, dict) else "") + if list_message: + messages[list_message] = messages.get(list_message, 0) + 1 + status_code = row.get("status_code") if isinstance(row, dict) else None + if isinstance(status_code, int): + statuses[str(status_code)] = statuses.get(str(status_code), 0) + 1 + if cause == "concurrency": + concurrency_count += 1 + model = str(row.get("model") or "unknown") if isinstance(row, dict) else "unknown" + models[model] = models.get(model, 0) + 1 + created_at = str(row.get("created_at") or "") if isinstance(row, dict) else "" + hour = created_at[:13] + ":00" if len(created_at) >= 13 else "unknown" + hours[hour] = hours.get(hour, 0) + 1 + stream = row.get("stream") if isinstance(row, dict) else None + stream_key = "stream" if stream is True else "synchronous" if stream is False else "unknown" + stream_modes[stream_key] += 1 + sample = { + "errorId": error_id, + "createdAt": row.get("created_at") if isinstance(row, dict) else None, + "requestId": row.get("request_id") if isinstance(row, dict) else None, + "model": row.get("model") if isinstance(row, dict) else None, + "stream": row.get("stream") if isinstance(row, dict) else None, + "cause": cause, + "evidence": evidence, + } + entry["sample"] = sample + analyzed_count = len(analyzed) + if analyzed_count == 0: + assessment = "insufficient-evidence" + elif concurrency_count == analyzed_count: + assessment = "concurrency-related" + elif concurrency_count > 0: + assessment = "mixed" + else: + assessment = "no-direct-concurrency-evidence" + dominant_cause = None + dominant_count = 0 + if causes: + dominant_cause, dominant_count = max(causes.items(), key=lambda pair: pair[1]) + dominant_share = round(dominant_count / analyzed_count, 4) if analyzed_count > 0 else None + if dominant_cause == "upstream-api-key-validation": + finding = "The upstream gateway returned HTTP 500 while validating its inbound API key; investigate its key lookup and storage dependencies first." + elif dominant_cause == "concurrency": + finding = "Direct concurrency-limit messages dominate the observed Ops records." + elif dominant_cause is not None: + finding = "The dominant observed Ops cause is " + dominant_cause + "." + else: + finding = "No structured HTTP 500 Ops records were available for cause analysis." + if concurrency_count > 0: + concurrency_action = "consider-lower-limit-after-controlled-comparison" + else: + concurrency_action = "do-not-lower-limit-as-primary-fix" + ranked_messages = sorted(messages.items(), key=lambda pair: (-pair[1], pair[0])) + message_limit = 20 if PAYLOAD.get("full") is True else 5 + return { + "source": "ops-upstream-errors", + "status": "available", + "logObservedStatusCodes": log_statuses, + "opsRecordCount": total, + "analyzedRecordCount": analyzed_count, + "detailRecordCount": min(detail_limit, analyzed_count), + "recordsTruncated": total > analyzed_count, + "statusBuckets": [{"statusCode": int(code), "count": count} for code, count in sorted(statuses.items(), key=lambda pair: int(pair[0]))], + "causeBuckets": [{"cause": cause, "count": count} for cause, count in sorted(causes.items())], + "messageBuckets": [{"message": message, "count": count} for message, count in ranked_messages[:message_limit]], + "concurrencyRelatedRecordCount": concurrency_count, + "assessment": assessment, + "rootCause": { + "dominantCause": dominant_cause, + "dominantCount": dominant_count, + "dominantShare": dominant_share, + "finding": finding, + }, + "errorRateEvidence": { + "errorRecordCount": total, + "requestAttemptDenominator": None, + "errorRate": None, + "reason": "The available Ops endpoint lists errors but does not provide all routing attempts for this account.", + }, + "concurrencyRecommendation": { + "action": concurrency_action, + "reason": "No error-time concurrency history is available; only explicit concurrency error text is treated as direct evidence.", + }, + "runtimeConcurrency": { + "limit": account.get("concurrency"), + "current": account.get("current_concurrency"), + "loadFactor": account.get("load_factor"), + "historicalAtErrorAvailable": False, + }, + **({ + "modelBuckets": [{"model": model, "count": count} for model, count in sorted(models.items(), key=lambda pair: (-pair[1], pair[0]))], + "hourBuckets": [{"hour": hour, "count": count} for hour, count in sorted(hours.items())], + "streamBuckets": [{"mode": mode, "count": count} for mode, count in stream_modes.items() if count > 0], + "samples": [entry["sample"] for entry in analyzed[:detail_limit]], + } if PAYLOAD.get("full") is True else { + "samples": [entry["sample"] for entry in analyzed[:1]], + }), + "disclosure": "Ops records are independently sampled evidence and are not positionally joined to account_upstream_error log lines.", + } + +def gateway_request_path(value): + path = str(value or "").split("?", 1)[0].rstrip("/") + return path in ( + "/responses", "/v1/responses", "/responses/compact", "/v1/responses/compact", + "/messages", "/v1/messages", "/chat/completions", "/v1/chat/completions", + ) + +def native_ops_snapshot(token, group_id, platform=None): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + base_query = f"group_id={group_id}&start_time={start_time}{platform_query}" + + def optional(path, label): + response = curl_api("GET", path, bearer=token) + try: + return {"status": "available", "data": data_of(response, label)} + except Exception as exc: + return {"status": "unavailable", "reason": compact_error(str(exc))} + + overview_result = optional(f"/api/v1/admin/ops/dashboard/overview?{base_query}&query_mode=raw", "get ops dashboard overview") + availability_result = optional(f"/api/v1/admin/ops/account-availability?group_id={group_id}{platform_query}", "get ops account availability") + concurrency_result = optional(f"/api/v1/admin/ops/concurrency?group_id={group_id}{platform_query}", "get ops concurrency") + sink_result = optional("/api/v1/admin/ops/system-logs/health", "get ops system log health") + + overview = overview_result.get("data") if overview_result.get("status") == "available" else None + overview_summary = None + if isinstance(overview, dict): + overview_summary = { + key: overview.get(key) + for key in ( + "start_time", "end_time", "health_score", "success_count", "error_count_total", + "business_limited_count", "error_count_sla", "request_count_total", "request_count_sla", + "error_rate", "upstream_error_rate", "upstream_error_count_excl_429_529", + "upstream_429_count", "upstream_529_count", + ) + } + + availability = availability_result.get("data") if availability_result.get("status") == "available" else None + availability_summary = None + if isinstance(availability, dict): + group_values = availability.get("group") if isinstance(availability.get("group"), dict) else {} + account_values = availability.get("account") if isinstance(availability.get("account"), dict) else {} + selected_group = group_values.get(str(group_id)) or group_values.get(group_id) + unavailable = [] + for value in account_values.values(): + if not isinstance(value, dict) or value.get("group_id") != group_id or value.get("is_available") is True: + continue + unavailable.append({ + "accountId": value.get("account_id"), + "accountName": value.get("account_name"), + "status": value.get("status"), + "rateLimited": value.get("is_rate_limited"), + "rateLimitRemainingSeconds": value.get("rate_limit_remaining_sec"), + "overloaded": value.get("is_overloaded"), + "overloadRemainingSeconds": value.get("overload_remaining_sec"), + "hasError": value.get("has_error"), + "error": compact_error(value.get("error_message")), + }) + availability_summary = { + "enabled": availability.get("enabled"), + "timestamp": availability.get("timestamp"), + "group": selected_group, + "unavailableAccounts": unavailable[:20 if PAYLOAD.get("full") is True else 8], + "unavailableAccountCount": max( + len(unavailable), + max(0, selected_group.get("total_accounts") - selected_group.get("available_count")) + if isinstance(selected_group, dict) + and isinstance(selected_group.get("total_accounts"), int) + and isinstance(selected_group.get("available_count"), int) + else 0, + ), + "unavailableAccountDetailsComplete": ( + not isinstance(selected_group, dict) + or not isinstance(selected_group.get("total_accounts"), int) + or not isinstance(selected_group.get("available_count"), int) + or len(unavailable) >= max(0, selected_group.get("total_accounts") - selected_group.get("available_count")) + ), + } + + concurrency = concurrency_result.get("data") if concurrency_result.get("status") == "available" else None + concurrency_summary = None + if isinstance(concurrency, dict): + group_values = concurrency.get("group") if isinstance(concurrency.get("group"), dict) else {} + account_values = concurrency.get("account") if isinstance(concurrency.get("account"), dict) else {} + selected_group = group_values.get(str(group_id)) or group_values.get(group_id) + loaded = [] + for value in account_values.values(): + if not isinstance(value, dict) or value.get("group_id") != group_id: + continue + if (value.get("current_in_use") or 0) <= 0 and (value.get("waiting_in_queue") or 0) <= 0: + continue + loaded.append({ + "accountId": value.get("account_id"), + "accountName": value.get("account_name"), + "currentInUse": value.get("current_in_use"), + "maxCapacity": value.get("max_capacity"), + "loadPercentage": value.get("load_percentage"), + "waitingInQueue": value.get("waiting_in_queue"), + }) + concurrency_summary = { + "enabled": concurrency.get("enabled"), + "timestamp": concurrency.get("timestamp"), + "group": selected_group, + "activeAccounts": sorted(loaded, key=lambda item: (-(item.get("loadPercentage") or 0), str(item.get("accountName") or "")))[:20 if PAYLOAD.get("full") is True else 8], + } + + sink = sink_result.get("data") if sink_result.get("status") == "available" else None + return { + "source": "sub2api-native-admin-ops", + "overview": {"status": overview_result.get("status"), "summary": overview_summary, "reason": overview_result.get("reason")}, + "accountAvailability": {"status": availability_result.get("status"), "summary": availability_summary, "reason": availability_result.get("reason")}, + "concurrency": {"status": concurrency_result.get("status"), "summary": concurrency_summary, "reason": concurrency_result.get("reason")}, + "systemLogSink": {"status": sink_result.get("status"), "health": sink if isinstance(sink, dict) else None, "reason": sink_result.get("reason")}, + } + +def all_group_error_overview(token, groups): + page_size = 20 + cursor = int(PAYLOAD.get("pageToken") or 0) + ordered = sorted( + [item for item in groups if isinstance(item, dict) and isinstance(item.get("id"), int)], + key=lambda item: item["id"], + ) + remaining = [item for item in ordered if item["id"] > cursor] + page = remaining[:page_size] + summaries = [] + totals = { + "requestCount": 0, + "errorCount": 0, + "businessLimitedCount": 0, + "upstreamErrorCount": 0, + "groupsNeedingAttention": 0, + } + for group in page: + group_id = group["id"] + platform = PAYLOAD.get("platform") or group.get("platform") + snapshot = native_ops_snapshot(token, group_id, platform) + overview = ((snapshot.get("overview") or {}).get("summary") or {}) + availability = ((snapshot.get("accountAvailability") or {}).get("summary") or {}) + availability_group = availability.get("group") if isinstance(availability.get("group"), dict) else {} + concurrency = ((snapshot.get("concurrency") or {}).get("summary") or {}) + concurrency_group = concurrency.get("group") if isinstance(concurrency.get("group"), dict) else {} + request_count = overview.get("request_count_total") + error_count = overview.get("error_count_total") + business_limited = overview.get("business_limited_count") + upstream_error_count = sum( + value for value in ( + overview.get("upstream_error_count_excl_429_529"), + overview.get("upstream_429_count"), + overview.get("upstream_529_count"), + ) if isinstance(value, int) + ) + unavailable_count = availability.get("unavailableAccountCount") + needs_attention = ( + (isinstance(error_count, int) and error_count > 0) + or upstream_error_count > 0 + or (isinstance(unavailable_count, int) and unavailable_count > 0) + ) + totals["requestCount"] += request_count if isinstance(request_count, int) else 0 + totals["errorCount"] += error_count if isinstance(error_count, int) else 0 + totals["businessLimitedCount"] += business_limited if isinstance(business_limited, int) else 0 + totals["upstreamErrorCount"] += upstream_error_count + totals["groupsNeedingAttention"] += 1 if needs_attention else 0 + summaries.append({ + "groupId": group_id, + "groupName": group.get("name"), + "platform": platform, + "status": group.get("status"), + "requestCount": request_count, + "errorCount": error_count, + "errorRate": overview.get("error_rate"), + "upstreamErrorCount": upstream_error_count, + "upstreamErrorRate": overview.get("upstream_error_rate"), + "businessLimitedCount": business_limited, + "totalAccounts": availability_group.get("total_accounts"), + "availableCount": availability_group.get("available_count"), + "rateLimitCount": availability_group.get("rate_limit_count"), + "errorAccountCount": availability_group.get("error_count"), + "unavailableAccountCount": unavailable_count, + "currentInUse": concurrency_group.get("current_in_use"), + "maxCapacity": concurrency_group.get("max_capacity"), + "waitingInQueue": concurrency_group.get("waiting_in_queue"), + "needsAttention": needs_attention, + "opsStatus": { + "overview": (snapshot.get("overview") or {}).get("status"), + "accountAvailability": (snapshot.get("accountAvailability") or {}).get("status"), + "concurrency": (snapshot.get("concurrency") or {}).get("status"), + }, + }) + next_token = str(page[-1]["id"]) if len(remaining) > page_size and page else None + totals["errorRate"] = round(totals["errorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None + totals["upstreamErrorRate"] = round(totals["upstreamErrorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None + return { + "scope": "all-groups-overview", + "window": {"since": PAYLOAD.get("since")}, + "platform": PAYLOAD.get("platform"), + "totalGroups": len(ordered), + "returnedGroups": len(summaries), + "pageSize": page_size, + "pageToken": str(cursor) if cursor > 0 else None, + "nextPageToken": next_token, + "pageTotals": totals, + "groups": summaries, + "disclosure": "Page totals cover only returned groups. Follow nextPageToken for remaining groups and use --group for account root causes and request indexes.", + } + +def native_system_log_lines(token, platform=None): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + markers = ( + "account_temp_unschedulable", "openai.upstream_failover_switching", + "openai.account_select_failed", "openai.forward_failed", "http request completed", + ) + rows_by_id = {} + queries = [] + for marker in markers: + path = f"/api/v1/admin/ops/system-logs?page=1&page_size=200&start_time={start_time}&q={quote(marker, safe='')}{platform_query}" + response = curl_api("GET", path, bearer=token) + try: + data = data_of(response, "list ops system logs") + except Exception as exc: + return None, { + "status": "unavailable", + "source": "sub2api-native-admin-ops-system-logs", + "reason": compact_error(str(exc)), + "fallback": "runtime-container-logs", + } + rows = items_of(data) + total = data.get("total", len(rows)) if isinstance(data, dict) else len(rows) + queries.append({"marker": marker, "returned": len(rows), "total": total, "truncated": total > len(rows)}) + for row in rows: + if not isinstance(row, dict): + continue + row_id = row.get("id") + key = str(row_id) if row_id is not None else json.dumps(row, sort_keys=True, default=str) + rows_by_id[key] = row + lines = [] + for row in sorted(rows_by_id.values(), key=lambda item: str(item.get("created_at") or "")): + extra = dict(row.get("extra") or {}) if isinstance(row.get("extra"), dict) else {} + for source_key, target_key in ( + ("request_id", "request_id"), ("client_request_id", "client_request_id"), + ("account_id", "account_id"), ("platform", "platform"), ("model", "model"), + ): + if extra.get(target_key) is None and row.get(source_key) is not None: + extra[target_key] = row.get(source_key) + message = str(row.get("message") or "") + lines.append(str(row.get("created_at") or "") + "\\t" + message + "\\t" + json.dumps(extra, separators=(",", ":"), default=str)) + return lines, { + "status": "available", + "source": "sub2api-native-admin-ops-system-logs", + "uniqueEventCount": len(lines), + "queries": queries, + "fallbackUsed": False, + } + +def policy_effectiveness(lines, accounts, source, group_id=None): + account_names = {str(item.get("id")): item.get("name") for item in accounts} + account_ids = set(account_names.keys()) + requests = {} + counts = { + "tempUnschedulableEvents": 0, + "failoverEvents": 0, + "selectFailedEvents": 0, + "forwardFailedEvents": 0, + "gatewayCompletedEvents": 0, + "gatewayFinalErrorEvents": 0, + } + missing_request_ids = { + "tempUnschedulable": 0, + "failover": 0, + "selectFailed": 0, + "forwardFailed": 0, + "gatewayCompleted": 0, + } + temp_accounts = {} + temp_samples = [] + failover_samples = [] + select_failed_samples = [] + final_statuses = {} + event_line_count = 0 + + def request_entry(request_id): + if not isinstance(request_id, str) or not request_id: + return None + if request_id not in requests: + requests[request_id] = { + "requestId": request_id, + "tempUnschedulableCount": 0, + "failoverCount": 0, + "selectFailedCount": 0, + "forwardFailedCount": 0, + "finalStatus": None, + } + return requests[request_id] + + for line in lines: + if not any(marker in line for marker in ( + "account_temp_unschedulable", "openai.upstream_failover_switching", + "openai.account_select_failed", "openai.forward_failed", "http request completed", + )): + continue + json_start = line.find("{") + if json_start < 0: + continue + try: + item = json.loads(line[json_start:]) + except json.JSONDecodeError: + continue + if not isinstance(item, dict): + continue + at = line.split("\\t", 1)[0].strip() or None + request_id = item.get("request_id") + account_id = item.get("account_id") + event_group_id = item.get("group_id") + if event_group_id is not None and group_id is not None and str(event_group_id) != str(group_id): + continue + if account_id is not None and str(account_id) not in account_ids: + continue + explicitly_scoped = account_id is not None or event_group_id is not None + if not explicitly_scoped and (not isinstance(request_id, str) or request_id not in requests): + continue + request = request_entry(request_id) + base = { + "at": at, + "requestId": request_id, + "accountId": account_id, + "accountName": account_names.get(str(account_id)), + } + if "account_temp_unschedulable" in line: + event_line_count += 1 + counts["tempUnschedulableEvents"] += 1 + if request is not None: + request["tempUnschedulableCount"] += 1 + else: + missing_request_ids["tempUnschedulable"] += 1 + key = str(account_id) if account_id is not None else "unknown" + if key not in temp_accounts: + temp_accounts[key] = { + "accountId": account_id, + "accountName": account_names.get(str(account_id)), + "count": 0, + } + temp_accounts[key]["count"] += 1 + if len(temp_samples) < 20: + temp_samples.append({ + **base, + "statusCode": item.get("status_code"), + "ruleIndex": item.get("rule_index"), + "matchedKeyword": item.get("matched_keyword"), + "until": item.get("until") or item.get("temp_unschedulable_until"), + "reason": compact_error(item.get("reason") or item.get("error")), + }) + elif "openai.upstream_failover_switching" in line: + event_line_count += 1 + counts["failoverEvents"] += 1 + if request is not None: + request["failoverCount"] += 1 + else: + missing_request_ids["failover"] += 1 + if len(failover_samples) < 20: + failover_samples.append({ + **base, + "switchCount": item.get("switch_count"), + "maxSwitches": item.get("max_switches"), + }) + elif "openai.account_select_failed" in line: + event_line_count += 1 + counts["selectFailedEvents"] += 1 + if request is not None: + request["selectFailedCount"] += 1 + else: + missing_request_ids["selectFailed"] += 1 + if len(select_failed_samples) < 20: + select_failed_samples.append({ + **base, + "excludedAccountCount": item.get("excluded_account_count"), + "error": compact_error(item.get("error")), + }) + elif "openai.forward_failed" in line: + event_line_count += 1 + counts["forwardFailedEvents"] += 1 + if request is not None: + request["forwardFailedCount"] += 1 + else: + missing_request_ids["forwardFailed"] += 1 + elif "http request completed" in line and gateway_request_path(item.get("path")): + event_line_count += 1 + counts["gatewayCompletedEvents"] += 1 + status_code = item.get("status_code") + if request is not None and isinstance(status_code, int): + request["finalStatus"] = status_code + elif request is None: + missing_request_ids["gatewayCompleted"] += 1 + if isinstance(status_code, int): + key = str(status_code) + final_statuses[key] = final_statuses.get(key, 0) + 1 + if status_code >= 400: + counts["gatewayFinalErrorEvents"] += 1 + + correlated = [item for item in requests.values() if item["failoverCount"] > 0] + succeeded = [item for item in correlated if isinstance(item.get("finalStatus"), int) and item["finalStatus"] < 400] + failed = [item for item in correlated if isinstance(item.get("finalStatus"), int) and item["finalStatus"] >= 400] + incomplete = [item for item in correlated if not isinstance(item.get("finalStatus"), int)] + cooled_then_failover = [item for item in correlated if item["tempUnschedulableCount"] > 0] + forward_with_http_success = [ + item for item in requests.values() + if item["forwardFailedCount"] > 0 and isinstance(item.get("finalStatus"), int) and item["finalStatus"] < 400 + ] + forward_with_http_error = [ + item for item in requests.values() + if item["forwardFailedCount"] > 0 and isinstance(item.get("finalStatus"), int) and item["finalStatus"] >= 400 + ] + forward_outcome_missing = [ + item for item in requests.values() + if item["forwardFailedCount"] > 0 and not isinstance(item.get("finalStatus"), int) + ] + sample_limit = 20 if PAYLOAD.get("full") is True else (2 if PAYLOAD.get("account") is not None else 5) + completed = counts["gatewayCompletedEvents"] + final_errors = counts["gatewayFinalErrorEvents"] + result = { + "source": source, + "scope": "group-membership-window", + "attribution": "Events with group_id are exact. Events without group_id are attributed by account membership and request_id correlation; shared-account events can appear in more than one group drilldown.", + "eventLineCount": event_line_count, + **counts, + "tempUnschedulableAccounts": sorted(temp_accounts.values(), key=lambda item: (-item["count"], str(item.get("accountName") or ""))), + "failoverRequestCount": len(correlated), + "failoverSucceededRequestCount": len(succeeded), + "failoverFailedRequestCount": len(failed), + "failoverOutcomeMissingRequestCount": len(incomplete), + "cooledThenFailoverRequestCount": len(cooled_then_failover), + "forwardFailureWithHttpSuccessRequestCount": len(forward_with_http_success), + "forwardFailureWithHttpErrorRequestCount": len(forward_with_http_error), + "forwardFailureOutcomeMissingRequestCount": len(forward_outcome_missing), + "requestIdCoverage": { + "missingByEventType": missing_request_ids, + "tempToFailoverCorrelationAvailable": counts["tempUnschedulableEvents"] > missing_request_ids["tempUnschedulable"], + }, + "gatewayFinalStatusBuckets": [ + {"statusCode": int(code), "count": count} + for code, count in sorted(final_statuses.items(), key=lambda pair: int(pair[0])) + ], + "gatewayOutcomeEvidence": { + "completedLogEventCount": completed, + "finalErrorLogEventCount": final_errors, + "observedFinalErrorShare": round(final_errors / completed, 4) if completed > 0 else None, + "denominator": "gateway http request completed log events for Responses, Messages, and Chat Completions paths", + "coverage": "since-window-and-tail-bounded", + "completeTrafficDenominatorGuaranteed": False, + }, + "requestDrilldown": { + "succeededAfterFailover": [item["requestId"] for item in succeeded[:sample_limit]], + "failedAfterFailover": [item["requestId"] for item in failed[:sample_limit]], + "outcomeMissingAfterFailover": [item["requestId"] for item in incomplete[:sample_limit]], + "selectFailed": [item["requestId"] for item in requests.values() if item["selectFailedCount"] > 0][:sample_limit], + "forwardFailureWithHttpSuccess": [item["requestId"] for item in forward_with_http_success[:sample_limit]], + "forwardFailureWithHttpError": [item["requestId"] for item in forward_with_http_error[:sample_limit]], + "forwardFailureOutcomeMissing": [item["requestId"] for item in forward_outcome_missing[:sample_limit]], + }, + "assessment": ( + "failover-observed-with-success" if succeeded else + "failover-observed-without-success" if correlated else + "no-failover-observed" + ), + "disclosure": "Rule matches require account_temp_unschedulable evidence; status codes alone are not treated as rule hits. Request outcomes are joined only by request_id. HTTP success can coexist with forward_failed on streaming responses and does not prove the client received a complete terminal event.", + } + if PAYLOAD.get("full") is True: + result["samples"] = { + "tempUnschedulable": temp_samples, + "failovers": failover_samples, + "selectFailures": select_failed_samples, + } + return result + +def observed_runtime_errors(token, accounts, group_id, platform=None): selector = PAYLOAD.get("account") selected = [item for item in accounts if selector is None or str(item.get("id")) == str(selector) or item.get("name") == selector] if selector is not None and len(selected) != 1: raise RuntimeError("account-not-found-or-ambiguous: " + str(selector)) - selected_ids = set(item.get("id") for item in selected) + selected_ids = set(str(item.get("id")) for item in selected) stats = {} for item in selected: - stats[item.get("id")] = { + stats[str(item.get("id"))] = { "accountName": item.get("name"), "accountId": item.get("id"), "upstreamEventCount": 0, @@ -430,8 +1438,50 @@ def observed_runtime_errors(accounts): "lastSeenAt": None, "errorSamples": [], } + native_ops = native_ops_snapshot(token, group_id, platform) + lines, system_log_source = native_system_log_lines(token, platform) + if lines is None: + lines = runtime_log_lines() + system_log_source["fallbackUsed"] = True + else: + marker_queries = [item for item in system_log_source.get("queries", []) if item.get("marker") != "http request completed"] + marker_total = sum(item.get("total", 0) for item in marker_queries if isinstance(item.get("total"), int)) + overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) + upstream_count = overview_summary.get("upstream_error_count_excl_429_529") or 0 + upstream_count += overview_summary.get("upstream_429_count") or 0 + upstream_count += overview_summary.get("upstream_529_count") or 0 + if marker_total == 0 and upstream_count > 0: + runtime_event_lines = [ + line for line in runtime_log_lines() + if any(marker in line for marker in ( + "account_temp_unschedulable", "openai.upstream_failover_switching", + "openai.account_select_failed", "openai.forward_failed", + )) + ] + lines.extend(runtime_event_lines) + system_log_source["source"] = "sub2api-native-admin-ops-with-runtime-event-fallback" + system_log_source["fallbackUsed"] = True + system_log_source["fallbackReason"] = "native system-log index returned no policy events while the native overview reported upstream errors" + system_log_source["fallbackEventCount"] = len(runtime_event_lines) + effectiveness = policy_effectiveness(lines, accounts, system_log_source.get("source"), group_id) + effectiveness["sourceStatus"] = system_log_source + overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) + request_total = overview_summary.get("request_count_total") + error_total = overview_summary.get("error_count_total") + if isinstance(request_total, int) and isinstance(error_total, int): + effectiveness["gatewayOutcomeEvidence"] = { + "source": "sub2api-native-admin-ops-dashboard-overview", + "requestCountTotal": request_total, + "errorCountTotal": error_total, + "errorRate": overview_summary.get("error_rate"), + "upstreamErrorRate": overview_summary.get("upstream_error_rate"), + "businessLimitedCount": overview_summary.get("business_limited_count"), + "startTime": overview_summary.get("start_time"), + "endTime": overview_summary.get("end_time"), + "completeTrafficDenominatorGuaranteed": True, + } scanned = 0 - for line in runtime_log_lines(): + for line in lines: if "account_upstream_error" not in line and "openai.forward_failed" not in line: continue json_start = line.find("{") @@ -442,11 +1492,12 @@ def observed_runtime_errors(accounts): except json.JSONDecodeError: continue account_id = item.get("account_id") if isinstance(item, dict) else None - if account_id not in selected_ids: + account_key = str(account_id) + if account_key not in selected_ids: continue scanned += 1 at = line.split("\t", 1)[0].strip() or None - current = stats[account_id] + current = stats[account_key] if at and (current["lastSeenAt"] is None or at > current["lastSeenAt"]): current["lastSeenAt"] = at if "account_upstream_error" in line: @@ -467,7 +1518,7 @@ def observed_runtime_errors(accounts): rows = [] aggregate = {} for item in selected: - current = stats[item.get("id")] + current = stats[str(item.get("id"))] observed = [{"statusCode": int(code), "count": count} for code, count in sorted(current["observedStatusCodes"].items(), key=lambda pair: int(pair[0]))] forwarded = [{"statusCode": int(code), "count": count} for code, count in sorted(current["forwardFailureStatusCodes"].items(), key=lambda pair: int(pair[0]))] for entry in observed: @@ -475,36 +1526,211 @@ def observed_runtime_errors(accounts): aggregate[key] = aggregate.get(key, 0) + entry["count"] current["observedStatusCodes"] = observed current["forwardFailureStatusCodes"] = forwarded + analysis = ops_error_analysis(token, item, observed) + has_ops_errors = analysis.get("status") == "available" and (analysis.get("opsRecordCount") or 0) > 0 + if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or has_ops_errors: + if PAYLOAD.get("full") is not True and analysis.get("status") == "available": + analysis = { + "source": analysis.get("source"), + "status": analysis.get("status"), + "opsRecordCount": analysis.get("opsRecordCount"), + "recordsTruncated": analysis.get("recordsTruncated"), + "statusBuckets": analysis.get("statusBuckets"), + "causeBuckets": analysis.get("causeBuckets"), + "rootCause": analysis.get("rootCause"), + "errorRateEvidence": analysis.get("errorRateEvidence"), + "concurrencyRecommendation": analysis.get("concurrencyRecommendation"), + "runtimeConcurrency": analysis.get("runtimeConcurrency"), + "samples": (analysis.get("samples") or [])[:1], + } + current["opsAnalysis"] = analysis if PAYLOAD.get("full") is not True: current["errorSamples"] = current["errorSamples"][:1] - if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or selector is not None: + if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or has_ops_errors or selector is not None: rows.append(current) + root_causes = {} + account_roots = [] + for row in rows: + analysis = row.get("opsAnalysis") + if not isinstance(analysis, dict) or analysis.get("status") != "available": + continue + for bucket in analysis.get("causeBuckets") or []: + cause = bucket.get("cause") + count = bucket.get("count") + if isinstance(cause, str) and isinstance(count, int): + root_causes[cause] = root_causes.get(cause, 0) + count + root = analysis.get("rootCause") or {} + account_roots.append({ + "accountId": row.get("accountId"), + "accountName": row.get("accountName"), + "dominantCause": root.get("dominantCause"), + "dominantCount": root.get("dominantCount"), + "dominantShare": root.get("dominantShare"), + }) + ranked_causes = sorted(root_causes.items(), key=lambda pair: (-pair[1], pair[0])) + native_overview = native_ops.get("overview") or {} + native_overview_values = native_overview.get("summary") if isinstance(native_overview.get("summary"), dict) else native_overview + native_error_rate_available = isinstance(native_overview_values.get("error_rate") if isinstance(native_overview_values, dict) else None, (int, float)) + account_count_with_errors = sum( + 1 for item in rows + if item["upstreamEventCount"] > 0 + or item["forwardFailureCount"] > 0 + or (((item.get("opsAnalysis") or {}).get("opsRecordCount") or 0) > 0) + ) + if selector is None and PAYLOAD.get("full") is not True: + for row in rows: + row.pop("opsAnalysis", None) + if PAYLOAD.get("full") is not True: + overview = ((native_ops.get("overview") or {}).get("summary") or {}) + availability = ((native_ops.get("accountAvailability") or {}).get("summary") or {}) + concurrency = ((native_ops.get("concurrency") or {}).get("summary") or {}) + sink = ((native_ops.get("systemLogSink") or {}).get("health") or {}) + unavailable_accounts = availability.get("unavailableAccounts") or [] + if selector is not None: + unavailable_accounts = [item for item in unavailable_accounts if item.get("accountId") in selected_ids] + native_ops = { + "source": native_ops.get("source"), + "overview": { + "status": (native_ops.get("overview") or {}).get("status"), + "healthScore": overview.get("health_score"), + "requestCount": overview.get("request_count_total"), + "errorCount": overview.get("error_count_total"), + "errorRate": overview.get("error_rate"), + "upstreamErrorRate": overview.get("upstream_error_rate"), + "businessLimitedCount": overview.get("business_limited_count"), + }, + "accountAvailability": { + "status": (native_ops.get("accountAvailability") or {}).get("status"), + "group": availability.get("group"), + "unavailableAccountCount": availability.get("unavailableAccountCount"), + "unavailableAccounts": unavailable_accounts, + }, + "concurrency": { + "status": (native_ops.get("concurrency") or {}).get("status"), + "group": concurrency.get("group"), + }, + "systemLogSink": { + "status": (native_ops.get("systemLogSink") or {}).get("status"), + "queueDepth": sink.get("queue_depth"), + "droppedCount": sink.get("dropped_count"), + "writeFailedCount": sink.get("write_failed_count"), + }, + } + source_status = effectiveness.get("sourceStatus") or {} + compact_source_status = { + "status": source_status.get("status"), + "source": source_status.get("source"), + "fallbackUsed": source_status.get("fallbackUsed"), + "fallbackReason": source_status.get("fallbackReason"), + "fallbackEventCount": source_status.get("fallbackEventCount"), + "nativeIndexedEventCounts": { + item.get("marker"): item.get("total") + for item in source_status.get("queries", []) + if isinstance(item, dict) + }, + } + drilldown = effectiveness.get("requestDrilldown") or {} + effectiveness = { + "source": effectiveness.get("source"), + "scope": effectiveness.get("scope"), + "attribution": effectiveness.get("attribution"), + "assessment": effectiveness.get("assessment"), + "ruleMatches": { + "tempUnschedulableEvents": effectiveness.get("tempUnschedulableEvents"), + "accounts": effectiveness.get("tempUnschedulableAccounts"), + }, + "failover": { + "events": effectiveness.get("failoverEvents"), + "requests": effectiveness.get("failoverRequestCount"), + "succeeded": effectiveness.get("failoverSucceededRequestCount"), + "failed": effectiveness.get("failoverFailedRequestCount"), + "outcomeMissing": effectiveness.get("failoverOutcomeMissingRequestCount"), + }, + "forwardFailure": { + "events": effectiveness.get("forwardFailedEvents"), + "withHttpSuccess": effectiveness.get("forwardFailureWithHttpSuccessRequestCount"), + "withHttpError": effectiveness.get("forwardFailureWithHttpErrorRequestCount"), + "outcomeMissing": effectiveness.get("forwardFailureOutcomeMissingRequestCount"), + }, + "selectFailedEvents": effectiveness.get("selectFailedEvents"), + "requestIdCoverage": effectiveness.get("requestIdCoverage"), + "gatewayOutcomeEvidence": effectiveness.get("gatewayOutcomeEvidence"), + "requestDrilldown": {key: value for key, value in drilldown.items() if value}, + "sourceStatus": compact_source_status, + "disclosure": effectiveness.get("disclosure"), + } + if selector is not None: + native_ops["accountAvailability"].pop("group", None) + native_ops.pop("concurrency", None) + native_ops.pop("systemLogSink", None) + effectiveness["sourceStatus"] = { + "source": compact_source_status.get("source"), + "fallbackUsed": compact_source_status.get("fallbackUsed"), + "fallbackReason": compact_source_status.get("fallbackReason"), + } return { "window": {"since": PAYLOAD.get("since"), "tail": PAYLOAD.get("tail"), "matchedLogEvents": scanned}, "aggregateObservedStatusCodes": [{"statusCode": int(code), "count": count} for code, count in sorted(aggregate.items(), key=lambda pair: int(pair[0]))], - "accountCountWithErrors": sum(1 for item in rows if item["upstreamEventCount"] > 0 or item["forwardFailureCount"] > 0), - "accounts": rows, + "accountCountWithErrors": account_count_with_errors, + "rootCauseAnalysis": { + "source": "ops-upstream-errors", + "scope": "account-window-for-group-members", + "attribution": "The upstream-errors endpoint is account scoped. If accounts belong to multiple groups, these cause buckets are shared evidence and must not be added across groups.", + "causeBuckets": [{"cause": cause, "count": count} for cause, count in ranked_causes], + "dominantCause": ranked_causes[0][0] if ranked_causes else None, + "dominantCount": ranked_causes[0][1] if ranked_causes else 0, + "accountRoots": account_roots, + "errorRateAvailable": native_error_rate_available, + }, + "nativeOps": native_ops, + "policyEffectiveness": effectiveness, + **( + {"accounts": rows} + if selector is not None or PAYLOAD.get("full") is True + else {"accountDrilldown": "rerun runtime errors with --account for per-account evidence"} + ), "message": "observedStatusCodes comes only from account_upstream_error; forward failures are supplemental message evidence", } def runtime_result(): token = login() - group = next((item for item in list_groups(token) if item.get("name") == PAYLOAD["groupName"]), None) + action = PAYLOAD["action"] + if action == "errors" and PAYLOAD.get("allGroups") is True: + groups = list_groups(token, PAYLOAD.get("platform")) + overview = all_group_error_overview(token, groups) + return { + "ok": True, + "operation": "errors", + "mutation": False, + "endpoint": "host-docker" if RUNTIME_MODE == "host-docker" else "k3s-service", + "allGroups": overview, + "valuesPrinted": False, + } + group_selector = PAYLOAD.get("group") + group_platform = PAYLOAD.get("platform") if action == "errors" else "openai" + if action == "errors" and group_selector is None: + group_platform = "openai" + groups = list_groups(token, group_platform) + selector = str(group_selector if group_selector is not None else PAYLOAD["groupName"]) + matching_groups = [item for item in groups if str(item.get("id")) == selector or item.get("name") == selector] + if len(matching_groups) > 1: + raise RuntimeError("group-selector-ambiguous: " + selector) + group = matching_groups[0] if matching_groups else None if not isinstance(group, dict) or group.get("id") is None: - raise RuntimeError("pool-group-not-found") - accounts = list_group_accounts(token, group["id"]) + raise RuntimeError("group-not-found: " + selector) + effective_platform = group_platform or group.get("platform") + accounts = list_group_accounts(token, group["id"], effective_platform) base = { - "group": {"id": group.get("id"), "name": group.get("name"), "status": group.get("status")}, + "group": {"id": group.get("id"), "name": group.get("name"), "platform": effective_platform, "status": group.get("status")}, "templates": [template_summary(item) for item in PAYLOAD["templates"]], "endpoint": "host-docker" if RUNTIME_MODE == "host-docker" else "k3s-service", "valuesPrinted": False, } - action = PAYLOAD["action"] if action == "list": details = [account_detail(token, item["id"]) for item in accounts] return {**base, "ok": True, "operation": "list", "mutation": False, "accountCount": len(details), "accounts": [account_summary(item) for item in sorted(details, key=lambda value: value.get("name") or "")]} if action == "errors": - errors = observed_runtime_errors(accounts) + errors = observed_runtime_errors(token, accounts, group["id"], effective_platform) return {**base, "ok": True, "operation": "errors", "mutation": False, "errors": errors} detail = find_account(token, accounts, str(PAYLOAD["account"])) if action == "get": diff --git a/scripts/src/platform-infra-sub2api-codex/types.ts b/scripts/src/platform-infra-sub2api-codex/types.ts index 347d4685..27e24ba1 100644 --- a/scripts/src/platform-infra-sub2api-codex/types.ts +++ b/scripts/src/platform-infra-sub2api-codex/types.ts @@ -348,9 +348,9 @@ export function codexPoolHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601", "bun scripts/cli.ts platform-infra sub2api codex-pool sync [--target D601] --confirm [--prune-removed]", "bun scripts/cli.ts platform-infra sub2api codex-pool validate [--target D601] [--full|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--full|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account [--target PK01] [--full|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--account ] [--since 24h] [--tail 50000] [--target PK01] [--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--json|--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account [--target PK01] [--json|--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--group |--all-groups] [--platform ] [--page-token ] [--account ] [--since 24h] [--tail 50000] [--target PK01] [--json|--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --account --template [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --account --kind temp-unschedulable [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]",