From 4b55fdb711249011e3db7968caf6738808da9365 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:25:13 +0200 Subject: [PATCH] feat: preserve parallel sub2api diagnostics work --- .../unidesk-sub2api/references/codex-pool.md | 18 + .../references/troubleshooting-accounts.md | 6 + .agents/skills/unidesk-webdev/SKILL.md | 7 + config/platform-infra/sub2api-codex-pool.yaml | 12 +- .../R2.2.10_Task_Report.md | 53 + .../R2.2.11_Task_Report.md | 68 + .../R2.2.12_Task_Report.md | 66 + .../R2.2.13_Task_Report.md | 61 + .../R2.2.15_Task_Report.md | 14 + .../R2.2.16_Task_Report.md | 23 + .../R2.2.18_Task_Report.md | 26 + .../R2.2.20_Task_Report.md | 42 + .../R2.2.22_Task_Report.md | 59 + .../R2.2.24_Task_Report.md | 47 + .../R2.2.8_Task_Report.md | 55 + .../R2.2.9_Task_Report.md | 48 + docs/MDTODO/sub2api-upstream-reliability.md | 54 + .../runtime-options.ts | 120 ++ .../platform-infra-sub2api-codex/runtime.ts | 1180 ++++++++++++++--- 19 files changed, 1789 insertions(+), 170 deletions(-) create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.10_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.11_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.12_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.13_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.15_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.16_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.18_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.20_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.22_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.24_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.8_Task_Report.md create mode 100644 docs/MDTODO/details/sub2api-upstream-reliability/R2.2.9_Task_Report.md create mode 100644 scripts/src/platform-infra-sub2api-codex/runtime-options.ts diff --git a/.agents/skills/unidesk-sub2api/references/codex-pool.md b/.agents/skills/unidesk-sub2api/references/codex-pool.md index 4b2969ca..2514dbc6 100644 --- a/.agents/skills/unidesk-sub2api/references/codex-pool.md +++ b/.agents/skills/unidesk-sub2api/references/codex-pool.md @@ -6,6 +6,11 @@ - PK01 路径不创建 k8s Secret、不部署 sentinel 资源，也不触发 `sub2api apply`、Docker compose、Caddy reload 或容器重启； - `runtime list|get` 查询 pool group 内全部账号或指定账号的实际状态；`get` 同时披露脱敏 `modelMapping` 和完整临时不可调度规则； - `runtime models --account` 通过 Sub2API 原生账号模型接口同时读取配置模型和上游实时模型，不把配置映射误当作真实上游能力； + - `runtime infrastructure --account` 只读关联账号 proxy 绑定与基础设施状态： + - 同时披露账号 proxy 记录和容器代理环境，不能因容器没有 `HTTP_PROXY` 就把账号误判为直连； + - 采集容器网络模式、DNS、默认路由、接口计数器、proxy socket、监听进程、服务状态和近期错误日志； + - 不发送上游账号测试或网络探针，不修改账号、proxy、调度或 YAML； + - socket、接口计数器和服务状态是查询时快照，不能单独证明历史故障窗口内网络正常； - `runtime errors` 优先使用 Sub2API 原生 admin/ops 能力： - `--all-groups` 按稳定 group ID 游标分页返回所有分组概要； - `--group ` 返回指定分组的账号根因、策略效果和 request 索引； @@ -17,6 +22,17 @@ - 客户错误画像提供模型、账号、状态码、同步/流式、阶段、入口和精确请求路径分布； - 原生接口没有账号维度 TTFT 时禁止从错误账号分布推导账号 TTFT； - 默认只给有界 request ID 索引，使用 `trace` 做精确下钻； + - 指定分组和 `--account` 下钻提供上游账号质量评分： + - 成功请求分母和账号级 TTFT 来自原生 admin usage； + - 客户错误来自原生 request-errors； + - 切号、同账号重试、临时不可调度和 `forward_failed` 来自原生 system-log 索引； + - 默认权重为可靠性 60、TTFT 25、当前可用性 15，并同时输出评分组成、等级、置信度和扣分原因； + - TTFT 少于 5 个样本时不进入分母，不能把缺失时延当成零时延； + - 至少 10 次账号尝试且至少 5 个 TTFT 样本时，才允许把高分映射为 A-D 可比较等级；样本不足的偶然高分标记为 `insufficient`； + - 至少 10 次账号尝试且评分低于 60 时，即使 TTFT 缺失也保留 E 级，避免明确的高失败率被“证据不足”掩盖； + - 请求证据不足时使用低置信度，不能据此自动调整 priority、capacity 或 schedulable； + - 账号名称是人类可读运维标识，应与账号 ID 一起完整输出；API Key、token 和 credentials 仍必须脱敏； + - 只有带 `group_id` 的 failover 和 `forward_failed` request ID 才进入分组账号失败率；账号级临时不可调度和同账号重试可以作为共享运维信号展示，但不加入分组失败分子； - 账号被多个分组共享时，账号级 upstream-errors 和缺少 `group_id` 的 runtime 事件属于共享证据，禁止跨分组相加； - dashboard overview 提供请求分母和错误率； - dashboard overview 的 `ttft` 提供成功流式请求的首 token 分位数，`duration` 提供成功请求总耗时分位数；二者不能互相替代； @@ -42,6 +58,7 @@ - 批量操作先校验全部 selector 存在且账号类型可修改，再顺序调用 Sub2API admin API： - 选择错误不会产生部分写入； - 上游 admin API 在批次中途失败时仍需逐账号 `get` 对账； + - runtime 模板必须按可复用错误语义设计，不得包含账号名称、账号 ID、provider URL 或分组特例；非 auth 错误使用精确状态码和稳定短语，auth 与账号状态错误保留专门处理路径； - 写操作不带 `--confirm` 时只返回 dry-run，带 `--confirm` 才通过 Sub2API admin API 写入； - 批量默认输出账号 ID、名称、change、mode、mutation 和汇总计数；runtime 命令不输出 API key 或完整 credentials，使用 `get`、`--full` 或 `--raw` 下钻； - 文档中的 runtime action 或参数被当前分支 CLI 拒绝时，按 CLI 集成漂移处理： @@ -58,6 +75,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool validate --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool runtime list --target PK01 bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --target PK01 --account bun scripts/cli.ts platform-infra sub2api codex-pool runtime models --target PK01 --account +bun scripts/cli.ts platform-infra sub2api codex-pool runtime infrastructure --target PK01 --account --since 30m bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h --all-groups bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target PK01 --since 24h --group diff --git a/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md b/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md index 461a3359..5591c9c1 100644 --- a/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md +++ b/.agents/skills/unidesk-sub2api/references/troubleshooting-accounts.md @@ -20,6 +20,12 @@ - 没有账号真实支持目标模型时，保持池不声明该模型并快速失败，不映射到名称相近的其他模型。 - 上游模型同步失败只能标记证据不足，不能据此补映射。 5. 调优按所有权下发： + - 非 auth 上游错误优先抽象为通用临时不可调度模板： + - 规则同时使用真实状态码和已证实的稳定响应短语； + - 先从当前 Sub2API 源码确认规则命中会在响应提交前触发当前请求 failover； + - 禁止在模板、CLI 或源码中硬编码账号名称、账号 ID、provider URL 或分组特例； + - 禁止用通用 400、通用 `invalid request` 或宽泛关键词覆盖真实客户参数错误； + - auth、token revoked、余额不足、分组删除和凭据失效使用专门的认证与账号状态路径，不混入通用非 auth 模板； - 通用模板先修改 owning YAML，并保持冷却保守；默认从一分钟开始，没有新证据和明确授权不得超过三分钟。 - YAML-managed 账号走 `sync`；runtime-manual 账号走显式 `runtime apply|delete`。 - 多账号使用 `--accounts` 精准集合；例外账号从 selector 中显式排除，不使用隐式全量操作。 diff --git a/.agents/skills/unidesk-webdev/SKILL.md b/.agents/skills/unidesk-webdev/SKILL.md index 856872fc..86f7c4d2 100644 --- a/.agents/skills/unidesk-webdev/SKILL.md +++ b/.agents/skills/unidesk-webdev/SKILL.md @@ -16,6 +16,13 @@ description: UniDesk Web 开发与浏览器验证技能。用户处理 UniDesk/H - 调试 HWLAB Cloud Web/Workbench 业务或功能 bug 时，使用 YAML 声明的 `--origin internal`；验收 public exposure、DNS、FRP、Caddy 或公网用户入口时，显式使用 `--origin public`。 - `--url` 只是 custom/local 一次性逃生口，与 `--origin` 互斥；禁止通过手写 URL 或 IP 选择内网/公网运行面。本地 fake-server、localhost、dist 静态服务和 custom URL 只能作为开发 preflight 证据。 - 禁止在本地或 master server 直接跑 `vue-tsc` / 前端全量 typecheck 作为默认验证；本地只做语法级检查和真实入口复测，完整类型检查交给 CI、PipelineRun 或明确指定的受控构建运行面。 +- HWLAB Cloud Web 的 Vitest 防误用规则： + - 以 HWLAB `docs/reference/web-test-resource-guard.md` 为权威； + - 配置必须默认关闭 watch、固定单 worker、限制 worker V8 堆； + - 测试、hook 和 teardown 必须设置有界 timeout； + - 正式 `test` 入口必须使用 `vitest run`、限制父进程 V8 堆并设置整体执行时限； + - Master server 只允许解析配置和检查 diff，不运行测试； + - Vitest 配置不能替代系统 cgroup `MemoryMax`。 - 桌面版 Web 截图、视觉复测和 web-probe 采样默认使用 `1920x1080` 视口；只有用户、issue 或 SPEC 明确指定其他尺寸时才覆盖，并在证据中写明覆盖原因。 - Web probe 命令和历史判定口径见 [references/web-probe.md](references/web-probe.md)；Playwright/fake-server 复现见 [references/playwright.md](references/playwright.md)。 - 前端改动遵循仓库既有设计系统、[references/design.md](references/design.md) 和 `$frontend-design` 全局 UI 规则；不要做营销式落地页替代真实工具页面。 diff --git a/config/platform-infra/sub2api-codex-pool.yaml b/config/platform-infra/sub2api-codex-pool.yaml index 9b9702ae..d1c1f343 100644 --- a/config/platform-infra/sub2api-codex-pool.yaml +++ b/config/platform-infra/sub2api-codex-pool.yaml @@ -17,9 +17,13 @@ runtime: enabled: true rules: - statusCode: 429 - keywords: [concurrency limit exceeded] + keywords: [concurrency limit exceeded, upstream rate limit exceeded] durationMinutes: 1 - description: 明确命中上游并发限制时短时冷却当前账号。 + description: 明确命中上游并发或速率限制时短时冷却当前账号。 + - statusCode: 400 + keywords: [input must be a list] + durationMinutes: 1 + description: 明确命中 Responses input 列表兼容错误时短时冷却并切换账号。 - statusCode: 500 keywords: [failed to validate api key, do request failed] durationMinutes: 1 @@ -32,6 +36,10 @@ runtime: keywords: [upstream service temporarily unavailable, upstream request failed, service temporarily unavailable, temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 description: 明确命中 503 上游暂时不可用、过载或并发限制时短时冷却当前账号。 + - statusCode: 504 + keywords: [concurrency limit exceeded, upstream service temporarily unavailable, input must be a list] + durationMinutes: 1 + description: 明确命中 504 上游并发限制、暂时不可用或 Responses input 列表兼容错误时短时冷却当前账号。 - statusCode: 524 keywords: [upstream request failed, service temporarily unavailable, overloaded, concurrency limit exceeded] durationMinutes: 1 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.10_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.10_Task_Report.md new file mode 100644 index 00000000..5e0a0715 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.10_Task_Report.md @@ -0,0 +1,53 @@ +# R2.2.10 任务报告 + +## 结论 + +- 最近 30 分钟复取时，账号 `lyon9801`（ID 15）共 37 次尝试、37 次成功、0 次失败，TTFT P95 为 9040 ms；“自用”分组唯一客户错误来自账号 27 的流式 504。因此 `lyon9801` 的 503 是更早的孤立事件，不是当前持续不稳定。 +- `lyon9801` 不是 Sub2API 直连账号。它绑定 proxy 3：`platform-infra-sub2api-pk01-local-egress-proxy`，HTTP endpoint 为 `127.0.0.1:10809`。实际路径是 PK01 `sub2api-app` → PK01 本地 Hysteria 客户端 → NC01 Hysteria 服务端 → OpenAI。 +- 代表请求 `da415526-b124-4e23-a4a7-ea48154c851e` 在请求开始约 1556485 ms 后收到上游 HTTP 503。Sub2API 随即进入 failover，但 1 ms 后选号因 `context canceled` 失败，最终返回 502。客户影响的主要放大器是错误发现过晚，切号时已无可执行上下文，而不是 failover 未实现。 +- 另一请求 `4bba7f9b-15e3-4891-b063-bd65fb6e3704` 在 HTTP 200 后出现上游 `response.failed`，现由 trace 正确标记为 `degraded / forward-failed-http-success`。这是响应已提交后的流内失败，不能安全切号，也不等同于 proxy 连接失败。 +- 当前证据更支持 OpenAI/Codex 上游返回 HTTP 503 或应用层流失败，不支持 PK01 DNS、proxy connect、TCP reset、容器重启、OOM 或 NC01 Hysteria 重启为主因。历史瞬时丢包仍不能凭当前快照绝对排除。 + +## 基础设施证据 + +- PK01 `sub2api-app`：running，restart 0，OOM false，host network。 +- 账号 proxy 3：active，`127.0.0.1:10809`；当前有 4 条 established 本地 TCP socket。 +- PK01 proxy owner：`hysteria`，`host-proxy-hysteria.service` active/running，restart 0；最近两小时 journal 未命中 reconnect、disconnect、timeout 或 error。 +- PK01 DNS 为 `127.0.0.53`，默认路由经 `eth0 / 10.0.8.1`；可见接口 RX/TX error/drop 均为 0。 +- NC01 服务端容器：`vpn-hysteria2`，镜像 `tobyxdd/hysteria:v2.9.3`，running，restart 0，OOM false，host network；运行文件实际位于 `/root/vpn-server`。故障时刻附近没有对应 tunnel disconnect/reconnect 日志。 +- 账号 15 的近期 Sub2API 网络错误索引没有 DNS failure、proxy connect failure、connection reset、unexpected EOF、broken pipe、connection refused、network unreachable 或 transport timeout。 + +## Sub2API v0.1.153 源码归因 + +- 账号存在 `ProxyID` 时，OpenAI forward 把账号 proxy URL 传给 HTTP upstream；HTTP proxy 通过 `http.Transport.Proxy` 生效。因此容器没有 `HTTP_PROXY` 环境变量不能证明账号直连。 +- HTTP 503 属于已收到 HTTP response 的路径；所有 5xx 会构造 `UpstreamFailoverError` 并触发账号切换。 +- proxy、DNS、TCP 或 TLS 在 HTTP round trip 前失败时走 transport-error 路径，Ops status 为 0，最终 failover error 为 502。账号 15 的代表证据是 upstream status 503，不符合该传输故障路径。 +- OpenAI response-header timeout 在 v0.1.153 默认是 0；PK01 运行实例也显示 disabled。它允许请求长时间等待，但现有 Ops 只有端到端耗时，不能把 1556485 ms 全部断言为等待响应头。 +- 客户 context 已取消时，源码明确不再 failover；这解释了 503 后 1 ms 即 `account_select_failed: context canceled`。 + +## CLI 增强 + +新增只读入口： + +```bash +bun scripts/cli.ts platform-infra sub2api codex-pool runtime infrastructure \ + --target PK01 --group 3 --account lyon9801 --since 30m +``` + +默认 Kubernetes 风格文本输出账号名称、proxy 绑定、容器状态、代理环境摘要、DNS、默认路由、接口计数器、socket、监听进程、服务状态和近期网络错误分类。该命令不发上游请求或网络探针，不修改账号、proxy、runtime、调度或 YAML；机器读取仍需显式 `--json`。 + +## 建议 + +- 暂不因这次孤立 503 降权或摘除 `lyon9801`；最近 30 分钟为 37/37 成功。 +- 下一步源头优化优先研究 OpenAI response-header timeout 与客户 deadline/failover 预算的配合，使 5xx 在客户上下文取消前暴露；这属于配置候选，当前未实验、未下发。 +- 对 HTTP 200 后 `response.failed`，应继续作为 degraded 客户可见错误单列；响应已提交后不能依靠换号掩盖。 +- 若再次发生，直接用 request ID 同时抓 trace 与 infrastructure，并保留故障时刻的 Hysteria per-session 指标；不要用故障后的 socket 快照替代历史证据。 + +## 验证 + +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts` +- `bun --check scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts` +- `bun --check scripts/src/output.ts` +- `bun test scripts/src/output.test.ts`：2 pass，0 fail。 +- `git diff --check` +- `runtime.ts`：2970 行，低于 3000 行硬上限。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.11_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.11_Task_Report.md new file mode 100644 index 00000000..74c36ae1 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.11_Task_Report.md @@ -0,0 +1,68 @@ +# R2.2.11 任务报告 + +## 范围与口径 + +- 只读复取 PK01 Sub2API 最近2小时所有分组，并以最近30分钟作为趋势对照。 +- 请求分母、客户错误和上游错误以 Sub2API 原生 admin/ops 为主；system-log 缺失时才使用 target runtime 日志。 +- 分组与账号数据在滚动窗口内持续变化，报告使用本轮最后一次完整分组快照；共享账号的 upstream cause 不跨组相加。 +- 未发送探针，未修改 runtime、YAML、proxy、账号或调度配置。 + +## 所有分组概要 + +最近2小时初始总览：1391请求、37个客户错误，客户错误率2.66%；74个上游错误，上游错误率5.32%。滚动窗口下钻时请求数增长到约1403，但客户错误总数仍为37。 + +- `unidesk-codex-pool`：约763请求、26客户错误，错误率3.41%；上游错误率约7.86%。 +- `自用`：约640请求、11客户错误，错误率1.72%；上游错误率约2.19%。 +- `grok`：0请求、0错误。 +- 两个活跃组均无排队，查询时并发约9至10/218；没有证据把主因归为容量不足或需要降低并发。 +- 账号29 `https://api.lwylink.xyz pro 0.1` 与账号30 `https://api.lwylink.xyz plus 0.05` 因余额不足403不可用；其余7个OpenAI账号当前可用。 + +最近30分钟对照：412请求、19客户错误，错误率4.61%；27个上游错误，上游错误率6.55%。其中： + +- `unidesk-codex-pool`：251请求、17客户错误，错误率6.77%。 +- `自用`：160请求、2客户错误，错误率1.25%。 +- 错误没有整体消退，而是更集中到 `unidesk-codex-pool`。 + +## 主因一：账号27的Responses兼容性400 + +账号27 `https://sub.yjxm1221.top plus 0.025` 是当前客户错误主因： + +- 2小时：139次尝试、91成功、48失败，失败率34.53%，评分38/E；26个分组客户错误中占24个。 +- 最近30分钟：48次尝试、30成功、18失败，失败率37.50%；17个客户错误全部来自账号27且全部是400。 +- 2小时客户错误状态分布中400有21个，另有404、503、524；模型以 `gpt-5.6-terra`、`gpt-5.6-luna`、`gpt-5.6-sol` 为主。 +- 代表请求 `1f3fe800-a72a-4383-88ba-e7311abb8534` 在630 ms内失败，原文为 `Input must be a list`。 +- 代表请求 `ee991d2f-08fe-4251-a509-c3d1a4bd8e45` 先经历账号33的524和账号28的503，切到账号27后返回 `Invalid 'input[213].id'`，最终502。 +- 另有 `invalid_id_prefix` 类流式 `response.failed`。这些更符合该上游对Responses长历史/input item格式的兼容缺陷，不是并发、网络或普通5xx不稳定。 +- 通用400不能直接加入临时不可调度或failover，否则会把真实客户参数错误错误切号；若后续治理，应只针对已证实的精确兼容错误短语和未提交响应阶段处理。 + +## 主因二：长等待后504，切号已无时间窗 + +`自用`组11个客户错误全部是 `gpt-5.6-sol`：8个504、2个503、1个524，10个为流式。 + +- `lyon9801` 2小时109次尝试、100成功、9失败，失败率8.26%，评分75.2/C；8个客户错误，8次failover全部失败。 +- 请求 `02f0cc91-6f7f-4106-b902-49ee5260ed57` 等待约29分31秒后才收到账号15的504；1 ms后选号因 `context canceled` 失败。 +- 最近30分钟 `lyon9801` 已恢复为34/34成功、TTFT P95约7.8秒；窗口内2个客户错误均来自账号27，不是账号15。 +- 当前 OpenAI response-header timeout 为disabled。长等待是客户影响放大器，但现有数据只有端到端耗时，不能断言全部时间都在等待响应头，也不能直接把观测下限当配置值。 + +## 上游隔离与切号效果 + +- 账号28 `https://ai.whistlelads.com plus 0.03`：2小时24次尝试全部触发失败，但有26次临时不可调度事件，分组客户错误为0；大量错误被换号吸收。最近30分钟7次失败中5次切号恢复、2次失败。 +- 账号31 `https://sub2.pokexiao.com plus 0.03`：2小时215次尝试仅1次失败，评分93.1/A；最近30分钟126次中124成功，2次failover均恢复。 +- 账号33 `https://api.iceiu.com plus 0.03`：2小时约3.6%失败，评分B；最近30分钟失败率升到14.55%，并有6次HTTP 200后forward failure，需要关注。 +- 成功切号样本 `4d305f0f-3e4f-4e0d-86d1-060a03820d22`：账号32返回524、账号28返回503，约14秒后由账号27完成并最终HTTP 200，证明5xx failover链路能工作。 + +## HTTP 200后的客户体验错误 + +- `unidesk-codex-pool` 2小时有12个、最近30分钟有6个 `forwardFailureWithHttpSuccess`。 +- `自用` 2小时有3个、最近30分钟有1个同类事件。 +- 请求 `e41f8745-e223-4fbf-9fb8-041a40c8ad94` 最终HTTP 200，但约417秒后出现 `stream data interval timeout`，trace正确标记为degraded。 +- 这类请求不会进入普通HTTP客户错误分子，因此37个客户错误不是全部体验损失；分组间存在共享账号证据，不能简单把所有forward failure相加成全局唯一请求数。 + +## 调优建议 + +1. 优先治理账号27的精确Responses兼容错误族，而不是降低全池并发：先按错误短语、请求阶段和模型统计；候选方案是对已证实的 `Input must be a list`、`Invalid input[*].id`、`invalid_id_prefix` 在响应未提交前触发failover或暂时避免把对应请求形态路由到该账号。是否需要Sub2API源码变更取决于现有精确classifier能否配置表达，不能用通用400规则替代。 +2. 研究上游超时与客户deadline的预算关系：目标是让504或无响应在客户context取消前暴露并留出切号时间。缺少客户deadline和response-header阶段耗时前，不直接给出安全timeout数值。 +3. 保留账号28现有短冷却策略；它已显著避免客户错误，不建议仅因上游失败率高而长期禁用。 +4. 对账号33继续短窗口观察，重点看HTTP 200后 `response.failed`/stream timeout；这类错误无法在响应已提交后安全换号。 +5. 账号29、30的余额不足属于静态不可用，应由账号所有者补充余额或退役；当前无排队，暂不构成容量事故。 +6. 不因2小时旧错误降权 `lyon9801`；最近30分钟已恢复。若再次出现长等待504，按request ID同步抓trace和基础设施快照。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.12_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.12_Task_Report.md new file mode 100644 index 00000000..53ebf28f --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.12_Task_Report.md @@ -0,0 +1,66 @@ +# R2.2.12 任务报告 + +## 结果 + +已为 runtime-manual 账号 `https://sub.yjxm1221.top plus 0.025`（ID 27）下发精确 `Input must be a list` 临时不可调度规则： + +- statusCode：400 +- keyword：`input must be a list` +- duration：1分钟 +- 行为：命中后当前账号短时冷却，当前请求返回 `UpstreamFailoverError` 进入切号。 + +未使用通用400匹配，避免把真实客户参数错误错误切号。 + +## 所有权与配置 + +- owning YAML：`config/platform-infra/sub2api-codex-pool.yaml` +- 模板：`codex-upstream-failover` +- 账号管理类型：`runtime-manual` +- 下发方式：精准 `runtime apply --account 27 --template codex-upstream-failover --confirm` +- 未把账号27加入 YAML managed profiles，也未运行全量 `sync`。 + +为避免模板覆盖账号27既有运行规则，YAML模板同时对齐了其已有： + +- 429 `upstream rate limit exceeded` +- 504 `concurrency limit exceeded` / `upstream service temporarily unavailable` + +最终规则从6条增至7条，保留429、500、502、503、504、524，仅新增精确400。 + +## 源码确认 + +Sub2API v0.1.153 普通400进入 `handleErrorResponse` 后会调用 `handleOpenAIAccountUpstreamError`；`RateLimitService.HandleUpstreamError` 在默认400处理前执行 `tryTempUnschedulable`。规则命中返回 `shouldDisable=true`，OpenAI gateway 构造 `UpstreamFailoverError`，因此既冷却后续调度，也会为当前请求触发切号。 + +## 执行证据 + +Dry-run： + +- account：`https://sub.yjxm1221.top plus 0.025` / 27 +- change：update +- beforeRules：6 +- afterRules：7 +- beforeCodes：429,500,502,503,504,524 +- afterCodes：400,429,500,502,503,504,524 + +确认下发：`mutation=true`。最终 `runtime get`： + +- management：runtime-manual +- schedulable：true +- matching rule count：7 +- status codes：400,429,500,502,503,504,524 +- pool mode：false + +## 短窗口 + +下发后5分钟只读窗口： + +- `unidesk-codex-pool`：43请求、0客户错误、客户错误率0%。 +- 账号27：7次尝试、7次成功、0失败。 +- 窗口内没有自然出现新的 `Input must be a list`，因此尚未观察到真实 `account_temp_unschedulable` 命中；不能把“零命中”解释为规则无效。 + +## 验证 + +- 精准 runtime apply dry-run通过。 +- 精准 runtime apply confirm通过。 +- 逐账号 runtime get对账通过。 +- `git diff --check`通过。 +- 未发送错误探针，未修改proxy、账号凭据、priority、capacity、membership或其他调度配置。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.13_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.13_Task_Report.md new file mode 100644 index 00000000..29e9b61a --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.13_Task_Report.md @@ -0,0 +1,61 @@ +# R2.2.13 任务报告 + +## 下发结果 + +已向PK01当前全部非 `lyon9801` OpenAI账号精准批量下发 `codex-upstream-failover`：显式selector为19、27、28、29、30、31、32、33、34。 + +- selected：9 +- changed：8 +- succeeded：8 +- failed：0 +- noop：账号27已提前匹配模板 +- `lyon9801`（15）：未进入selector，保持0条规则、无模板 + +最终账号19、27、28、29、30、31、32、33、34均为7条规则，状态码集合400、429、500、502、503、504、524，模板均为 `codex-upstream-failover`。账号29、30仍保持余额不足导致的error与schedulable=false；模板下发未恢复账号状态。 + +## CLI修复 + +原CLI文档声明支持 `--accounts`，但parser实际拒绝。已恢复通用精准批量入口： + +- 支持1至100个逗号分隔ID或精确名称； +- `--account`与`--accounts`互斥； +- 仅apply/delete接受批量selector； +- 先解析全部selector并校验账号类型，再进行任何写入； +- 默认Kubernetes风格批量计划与结果表； +- 参数解析拆到 `runtime-options.ts`，`runtime.ts`收敛到2915行。 + +Artificer后续任务R2.2.14 / issue #1998负责把确认写入后的自动批量回读对账内建到CLI，消除外部逐账号get。 + +## 真实效果 + +下发后首个5分钟窗口： + +- 约85请求； +- 5个上游错误； +- 0客户错误； +- `unidesk-codex-pool`：5次临时不可调度、3次failover、3次全部恢复； +- `自用`：2次failover、2次全部恢复。 + +请求 `290215cf-e32b-44df-b2d6-842298c7e690` 证明新400规则真实命中：账号27收到上游400，规则1将其冷却1分钟，当前请求切到账号33，4.055秒后最终HTTP 200，错误未暴露给客户。 + +10分钟窗口出现的请求 `68cd320a-4d2b-482e-93ed-284625096d8f` 是模板下发前开始的旧长请求：等待约719秒后502，切号时context canceled，不能据此判定新模板失效。 + +## Skill固化 + +账号排障reference已固化： + +- 非auth错误优先使用状态码加稳定短语的通用模板； +- 先确认响应提交前能触发当前请求failover； +- 禁止账号、ID、provider URL或分组硬编码； +- 禁止通用400或宽泛invalid request； +- auth/token/余额/凭据状态保留专门路径。 + +## 验证 + +- 批量dry-run：9个selector全部解析，8 update、1 noop。 +- 批量confirm：8 succeeded、0 failed。 +- 再次批量dry-run：9/9 noop。 +- 逐账号运行态对账完成；后续由R2.2.14把该过程内建到CLI。 +- `bun --check`通过。 +- `git diff --check`通过。 +- skill quick validate通过。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.15_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.15_Task_Report.md new file mode 100644 index 00000000..f4945720 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.15_Task_Report.md @@ -0,0 +1,14 @@ +# R2.2.15 任务报告 + +Artificer 已完成只读调查并把完整证据写入 [GitHub Issue #1999 评论](https://github.com/pikasTech/unidesk/issues/1999#issuecomment-4966948330)。 + +核心结论： + +- 精确“状态码 + 稳定短语”模板适合吸收可确认的非 auth 上游兼容错误；不得使用 generic 400、generic invalid request 或账号/provider 特例。 +- 400 `Input must be a list` 已有真实切号成功证据。本轮新请求 `e5c08dde-9e60-4a14-9458-5574cec00b65` 进一步证明：首跳账号 `https://sub.yjxm1221.top plus 0.025` 的 400 可冷却并切号，但第二跳账号 `https://sub2.pokexiao.com plus 0.03` 等待约 60.6 秒后返回 504，第二次 failover 时 context 已取消。 +- 主代理已把同一精确短语加入通用 504 规则并下发非 `lyon9801` 账号；这只降低后续重复命中，不能缩短首次长等待。 +- `Expected an ID that begins with 'ctc'` 与 `invalid_id_prefix` 暂不下发，需先有跨账号成功切换证据，避免把客户载荷或会话亲和错误误判为上游故障。 +- 根治首次长等待穿透需要 Sub2API 源码能力：客户总 deadline、动态每跳首字/响应头预算及 failover reserve；静态 response-header timeout 或 Caddy timeout 不能完整替代。 +- 流式响应一旦向客户提交字节，Sub2API 不能安全重放，只能终止流并记录错误。 + +本子任务未发送探针，未修改 runtime、YAML、账号、proxy、调度或源码，无 commit、PR 和 rollout。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.16_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.16_Task_Report.md new file mode 100644 index 00000000..919aad77 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.16_Task_Report.md @@ -0,0 +1,23 @@ +# R2.2.16 任务报告 + +## 结论 + +用户 `mail0795336304@163.com` 的“卡住约 15 分钟”不发生在 Sub2API 的在途请求、账号排队或 proxy 3 出网阶段。服务端证据显示，上一笔请求于 2026-07-14 14:43:33（UTC+8）成功结束，下一笔直到 14:59:54 才进入 Sub2API，中间约 16 分 21 秒没有该用户的新入站请求。卡点因此位于 Sub2API 入站之前，优先怀疑客户端、本地 Agent 状态机或客户端到入口之间的请求提交链路。 + +## 原生运维证据 + +- 用户 ID：22；账号状态 active；用户并发上限 10，调查时 current concurrency 为 0。 +- API Key：名称 `pika_win`、ID 46、分组 `unidesk-codex-pool`；调查时 current concurrency 为 0。 +- 最近 30 分钟共 7 笔请求，均为成功，无用户可见错误。 +- 14:43:33 完成请求 `client:1daa3f3e-99a5-4fc4-809c-6f6ef854a2b2`，流式、`gpt-5.6-sol`，耗时 28.922 秒，账号 33。 +- 14:59:54 下一笔请求 `client:8d1f8ef9-e703-482c-ace9-0d8f13a5c8e3` 才进入服务端；内部 request ID `efdb251f-8e86-4f18-99ee-fa1094419dc0`，15:00:43 完成，HTTP 200，耗时 49.115 秒。 +- 最新请求使用上游账号 `https://api.iceiu.com plus 0.03`（ID 33）、模型 `gpt-5.6-sol`、proxy 3；没有 failover、select failure 或 forward failure。 +- proxy 3 为 `platform-infra-sub2api-pk01-local-egress-proxy`，127.0.0.1:10809；对应 Hysteria 客户端自 7 月 8 日持续 active，调查窗口无错误日志，当前存在正常连接。 + +## 判定边界 + +Sub2API v0.1.153 当前原生 request 明细记录请求完成耗时，但未记录流式首字时间和逐事件间隔。因此不能从现有数据精确证明客户端何时展示首字；不过“16 分 21 秒无入站请求”、用户/API Key 并发均为 0，以及随后请求 49 秒正常完成，足以排除 Sub2API 内部排队、上游账号占槽 15 分钟和 proxy 3 在该窗口报错。 + +## 建议 + +让用户侧保留该时段客户端日志，重点检查本地 Agent 是否在上一轮完成后未推进状态、请求是否直到 14:59:54 才真正发出，以及本地 HTTP/SSE 客户端是否错误等待已经关闭的流。后续 CLI 可泛化补充按用户邮箱映射用户 ID、请求间空窗和 TTFT/末事件可见性；本次只读调查未取消请求，也未修改 runtime、YAML、proxy 或调度。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.18_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.18_Task_Report.md new file mode 100644 index 00000000..f1bff722 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.18_Task_Report.md @@ -0,0 +1,26 @@ +# R2.2.18 任务报告 + +## 结果 + +PK01 Sub2API 已通过 YAML-first 受控流程从 v0.1.153 升级到 v0.1.155。实际运行镜像为 `docker.1panel.live/weishaw/sub2api:0.1.155`，容器状态 healthy。 + +## Source of truth + +- owning YAML：`config/platform-infra/sub2api.yaml` +- 仅修改 PK01 target 的 `image.tag`：`0.1.153` → `0.1.155` +- PR：https://github.com/pikasTech/unidesk/pull/2003 +- merge commit：`d97605d8f20d3c87ff16f747cd7e500d79f9827b` +- 未修改其他 target、公开域名、Secret、Codex pool、账号 runtime 或调度策略。 + +## 受控部署证据 + +- `platform-infra sub2api plan --target PK01`：通过。 +- `platform-infra sub2api apply --target PK01 --dry-run`：通过，渲染目标镜像 v0.1.155。 +- `platform-infra sub2api image-prepull --target PK01 --confirm --wait`：成功；镜像 ID `sha256:ec8c1452f9dc5f4ca...`。 +- apply job：`platform_infra_sub2api_apply_pk01_20260714074034649_757de2`，状态 succeeded，exit 0。 +- `platform-infra sub2api status --target PK01`：app running/healthy，实际镜像 v0.1.155；Redis保持原容器运行。 +- `platform-infra sub2api validate --target PK01`：通过；app、Redis、外部 PostgreSQL、本地 health 与 PK01 Caddy 公网 health 均成功。 + +## 边界 + +本次是纯镜像升级，没有执行 `codex-pool plan|sync|validate`，没有对齐或覆盖手工账号、临时不可调度规则、proxy 绑定和其他 runtime 配置。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.20_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.20_Task_Report.md new file mode 100644 index 00000000..266d79c9 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.20_Task_Report.md @@ -0,0 +1,42 @@ +# R2.2.20 任务报告 + +## 目标 + +快速缓解客户错误 #17440（request ID `e5c08dde-9e60-4a14-9458-5574cec00b65`），只使用 YAML-first 通用临时不可调度模板和受控 runtime CRUD，不修改 Sub2API 源码、账号容量、优先级、代理或凭据。 + +## 根因证据 + +- 原生 trace 显示请求先由账号 27（`https://sub.yjxm1221.top plus 0.025`）返回 400，精确短语 `Input must be a list` 已命中既有 400 规则并触发第一次 failover。 +- 第二跳账号 31（`https://sub2.pokexiao.com plus 0.03`）等待约 60.6 秒后返回 504；响应 detail 仍为 `Input must be a list`。 +- 第二次 failover 已进入，但选号时客户 context 已取消，最终 `reason=failover-attempted-no-candidate`。因此客户可见 504 的直接原因是第二跳耗尽请求时间窗，而不是首跳 400 未切号。 +- 账号 31 在此前 30 分钟有 75 次尝试、8 次失败、失败率 10.67%，该窗口唯一客户错误就是本请求；2 小时窗口质量仍为 B，故不据单次事件调整长期容量或优先级。 + +## 配置调优 + +owning YAML：`config/platform-infra/sub2api-codex-pool.yaml`。 + +在通用模板 `codex-upstream-failover` 的精确 504 规则中加入关键词 `input must be a list`，冷却时间保持 1 分钟。该规则只匹配实际 504 与稳定短语，不泛化到其他 4xx/5xx，也不涉及 auth 错误。 + +通过显式 selector `19,27,28,29,30,31,32,33,34` 对 9 个非 `lyon9801` OpenAI 账号执行： + +1. 批量 dry-run：selected=9、changed=9、failed=0。 +2. 批量 confirm：selected=9、succeeded=9、failed=0。 +3. 回读账号 31 和 27：504 规则均包含 `input must be a list`，模板匹配正常。 +4. 回读 `lyon9801`：临时不可调度仍为 disabled、0 条规则，例外未被改变。 + +## 下发后观察 + +下发后的近期 10 分钟原生 Ops 窗口： + +- 请求 88； +- 客户错误 0，错误率 0.00%； +- failover 8 次，8 次均恢复成功； +- select failed 0； +- 临时不可调度命中 4 次； +- 当前排队 0。 + +该窗口尚未出现新的“504 + Input must be a list”样本，因此只能证明配置已实际生效且未见副作用，不能把短窗口零错误宣称为长期因果验证。 + +## 边界与剩余风险 + +本次规则会在同类 504 首次发生后短时摘除对应账号，减少连续客户请求重复命中；它无法挽救已经在第二跳耗尽 context 的当前请求。若同类首次错误仍频繁穿透，下一步应从 Sub2API 源码能力解决 failover 时间预算/每跳超时，而不是继续扩大关键词或盲目降低全池并发。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.22_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.22_Task_Report.md new file mode 100644 index 00000000..ddf9d433 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.22_Task_Report.md @@ -0,0 +1,59 @@ +# R2.2.22 任务报告 + +## 结论 + +错误 #17490（request ID `4eb9a84b-7346-47a8-bb66-fb7b44317279`）已确认由 Sub2API 内部渠道监控使用的 `monitor` API Key（ID 19）发起。账号 `https://sub2.pokexiao.com plus 0.03`（ID 31）长等待后返回 502；内部监控客户端已因响应头超时取消请求，Sub2API 却继续等待被去取消的上游请求。上游错误返回后虽然进入 failover，下一轮选号使用的原始 request context 已取消，无法开始第二跳，最终记录 502。 + +## 原生 trace + +- 请求：`/v1/responses`，`gpt-5.6-sol`，同步。 +- 调用方：用户 1、API Key `monitor`（ID 19）。 +- 单跳总耗时约 54,339 ms。 +- 上游状态：502，原始 detail：`error code: 502`。 +- Sub2API 记录 `openai.upstream_failover_switching`，switch 1/10。 +- 同一毫秒选号失败：`context canceled`，排除账号数 1。 +- 最终 HTTP 502，`reason=failover-attempted-no-candidate`。 + +近 30 分钟另一条客户错误 #17491（request ID `426551fe-cdd4-4563-a3d0-89b97aa5ffa3`）完全同型，但来自不同用户/IP，不是内部 monitor：同一账号和原始 502 detail，流式，单跳约 36,157 ms，failover 后立即 `context canceled`。因此内部监控 timeout 能解释 #17490，但不能解释所有账号 31 的客户穿透。 + +## 内部监控 timeout 与源码链 + +Sub2API v0.1.155 官方源码中，渠道监控 timeout 是 MVP 硬编码值： + +- `monitorResponseHeaderTimeout = 30s`：等待响应头； +- `monitorRequestTimeout = 45s`：单次模型请求总超时； +- runner 外层使用请求超时、PING 超时和缓冲构造总 context。 + +网关上游转发则调用 `detachUpstreamContext(ctx)`，其实现为 `context.WithoutCancel(ctx)`；failover 选号继续使用 `c.Request.Context()`。因此 #17490 的完整时间线是： + +1. 内部渠道监控发起请求； +2. 约 30 秒仍无响应头，监控 HTTP client 取消请求并记录 timeout； +3. 网关上游请求忽略该取消，继续等待账号 31； +4. 约 54.3 秒账号 31 返回 502； +5. handler 触发 failover，但原始监控 request context 已取消； +6. 选号立即 `context canceled`，没有第二跳。 + +内部监控的 30 秒响应头超时确实等不到本次切号，但根因不仅是 timeout 数值短，还包括“首轮上游去取消、后续选号绑定已取消 context”的生命周期不一致。 + +## 账号与策略 + +账号 31 调查窗口内 163 次尝试、149 次成功、14 次失败，失败率 8.59%，质量 D/69；14 次 failover 信号中 2 次客户穿透。现有模板未命中该账号的 502，临时不可调度计数为 0。 + +原因是规则匹配原始上游响应体。模板已有 `upstream service temporarily unavailable` 等关键词，但原始 body 只有 `error code: 502`；UI 包装消息不是规则匹配输入。 + +## 基础设施 + +账号 31 绑定 proxy 3：`platform-infra-sub2api-pk01-local-egress-proxy`，HTTP `127.0.0.1:10809`；owner `host-proxy-hysteria.service` active/running、0 重启。Sub2API app 0 restart、无 OOM，主机接口无 RX/TX error/drop。同一 proxy 的其他账号仍有成功请求。查询时快照不能排除历史抖动，但没有本地 proxy 整体故障证据。 + +## 调优候选 + +1. 内部监控：把响应头 timeout、总请求 timeout 和 cadence 从硬编码提升为原生可配置事实，并在 CLI 展示来源；timeout 必须与网关每跳 deadline/failover reserve 联合设计，不能只单独放大。 +2. 网关源码：统一上游与 failover 生命周期，记录 client deadline、hop deadline、remaining budget 和 cancel source。客户端断开后若无需继续工作，应取消首轮上游；若要内部完成 failover，则选号与下一跳必须使用有界内部 context，同时停止无意义的下游写回。 +3. 配置缓解：精确短语 `error code: 502` 可进入通用非 auth 502 临时不可调度模板，减少后续请求再次命中，但不能挽救首次请求。 +4. 不建议仅降低全池并发；窗口无排队，故障发生在单账号长等待。 + +本任务只读，未修改 runtime、YAML、账号、proxy、调度或 Sub2API 源码。 + +## 原生记录闭环 + +channel 3 记录 15589：checkedAt 2026-07-14T08:52:45Z，latency 30,011 ms，错误为 http2: timeout awaiting response headers；对应 #17490 从约 08:52:45Z 开始、网关在 08:53:39Z 结束，证明内部监控在 30 秒先取消，而去取消的首轮上游继续约 24 秒后才返回并尝试 failover。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.24_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.24_Task_Report.md new file mode 100644 index 00000000..073a9057 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.24_Task_Report.md @@ -0,0 +1,47 @@ +# R2.2.24 任务报告 + +## 结论 + +错误 #17511（请求 `f7ff0efe-64d3-4dd7-8716-20bdfc3a2f90`）不是内部 monitor 请求，也不是账号池没有候选。它由用户 1 的 `main` API Key（ID 20）发起，来源 IP `152.53.229.148`，分组“自用”（ID 3），流式 `/responses`。 + +账号 `https://sub2.pokexiao.com plus 0.03`（ID 31）等待 63,238 ms 后返回上游 502；Sub2API 在 1 ms 内记录 `openai.upstream_failover_switching`，随后选号立即以 `context canceled` 失败，最终 63,239 ms 返回 502。因此准确表述是：切号已触发，但切号开始时请求 context 已取消，备用账号没有被选取和调用。 + +同一 Key、同一来源 IP 的请求 `0503e439-ee9b-4954-bc2d-3087ec3a1d5e` 在此前 6 分钟出现相同链路：账号 31 等待 75,965 ms 后返回 504，切号同毫秒触发，选号立即 `context canceled`。这证明不是单次调度抖动。 + +## 运行面证据 + +- “自用”分组在窗口内有 9 个账号，其中 7 个可用，当前 18/218 并发，队列 0。 +- #17511 选号失败时仅排除账号 31，不存在“所有备用账号均被排除”。 +- 账号 31 最近窗口 286 次尝试、277 成功、9 失败，失败率 3.15%，质量分 89/B；9 次 failover 中 2 次最终客户错误。 +- OpenAI 专用响应头超时当前为 disabled，来源为进程环境。 +- 成功流式请求 TTFT P99 约 61.6 秒、最大约 92.9 秒；因此不能未经验证直接把全局响应头超时压到 45–60 秒。 + +## v0.1.155 源码归因 + +`openai_gateway_handler.go` 的 failover 循环： + +1. 选号使用 `c.Request.Context()`。 +2. 上游转发进入 `Forward(c.Request.Context(), ...)`。 +3. `Forward` 构建上游请求前调用 `detachUpstreamContext(ctx)`。 +4. `detachUpstreamContext` 返回 `context.WithoutCancel(ctx)`，官方测试 `TestDetachUpstreamContextIgnoresClientCancel` 明确验证客户端取消不会取消上游请求。 +5. 首轮上游最终返回可 failover 的 502/504 后，handler 回到循环，再次用原始且已取消的 `c.Request.Context()` 选号，于是立即失败。 + +这是“首轮上游去取消、后续选号仍绑定下游取消”的生命周期不一致。它允许客户端断开后首轮上游继续消耗时间，但不保留后续 failover 的执行窗口。 + +## 版本与配置判断 + +- v0.1.153、v0.1.152、v0.1.151、v0.1.150、v0.1.149 均存在相同 `context.WithoutCancel` 实现及调用，回退到这些版本不能修复该问题。 +- 公网 Caddy owning YAML 的 `responseHeaderTimeoutSeconds` 为 600 秒，明显长于本次 63 秒，不支持“Caddy 先截断”的假设。 +- 简单降低全局 `gateway.openai_response_header_timeout` 能让首轮更早失败并为切号留预算，但当前成功 TTFT 长尾已超过 60 秒，未经按请求类型/账号评估会误杀有效请求。 +- 精确匹配账号 31 的原始错误短语 `error code: 502` 并临时不可调度可减少后续请求再次命中，但不能挽救触发规则的首个请求，也不能修复 context 生命周期。 + +## 建议 + +1. 源码根治：统一上游请求与 failover 循环的生命周期。至少在进入下一轮前检测原始 context；若业务要求客户端断开后停止工作，则不要对上游使用 `WithoutCancel`。若业务要求内部继续完成 failover，则选号、转发和最终写回必须使用一个有界的内部 deadline，并在客户端已断开时停止无意义的继续写回。 +2. 增加 failover reserve：每次首字节等待不能耗尽整体预算，应根据请求类型（普通/compact）和剩余预算设置 per-hop deadline；日志需记录 client deadline、hop deadline、remaining budget 和 cancel source。 +3. 配置缓解前先用 CLI 按普通/compact、账号统计 TTFT 与错误等待时长，再选择高于正常长尾且低于客户端 deadline 的阈值；当前证据不足以安全给出单一秒数。 +4. 对账号 31 的稳定原始 502 短语使用通用非 auth 临时不可调度模板，只作为减少连续命中的缓解，不宣称根治。 + +## 官方 main 核对 + +2026-07-14 只读抓取官方 Wei-Shaw/sub2api main 后，detachUpstreamContext 仍返回 context.WithoutCancel(ctx)，Responses failover 选号仍使用 c.Request.Context()；官方最新主线尚未修复该生命周期不一致。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.8_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.8_Task_Report.md new file mode 100644 index 00000000..92c5cb27 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.8_Task_Report.md @@ -0,0 +1,55 @@ +# R2.2.8 任务报告 + +## 结果 + +已增强 Sub2API 只读诊断 CLI，按分组和上游账号归因近期请求质量。默认输出保持 Kubernetes 风格表格，完整展示账号名称和账号 ID；显式 `--json` 提供紧凑语义摘要，整组概要通过 `--account` 和 request ID 渐进下钻。未修改 Sub2API runtime、YAML、账号调度或源码运行面。 + +## 数据口径 + +- 成功请求、流式请求和账号 TTFT 来自 Sub2API 原生 `/api/v1/admin/usage`。 +- 客户可见错误来自原生 `/api/v1/admin/ops/request-errors`。 +- 切号、同账号重试、临时不可调度和 `forward_failed` 来自原生 system-log 索引，并自动分页读取近期窗口。 +- 失败分子按 request ID 去重；账号 attempts 是上游账号尝试次数，不等于客户请求数，切号时一个客户请求可以计入多个账号。 +- 评分权重为可靠性 60、TTFT 25、当前可用性 15，置信度单独计算。TTFT 少于 5 个样本不按零时延计分。 +- 至少 10 次尝试且至少 5 个 TTFT 样本时，才把高分映射为 A-D 可比较等级；样本不足的偶然高分标记为 `insufficient`。至少 10 次尝试且评分低于 60 时保留 E，避免高失败率被证据不足掩盖。 +- 账号名称是人类可读运维信息，报告和 CLI 完整输出；API Key、token、credentials 继续脱敏。 + +## 近期只读快照 + +采样窗口为 2026-07-14 05:32-05:35 UTC 向前 60 分钟，滚动值会随请求变化。 + +- 所有分组共 530 个请求、13 个客户错误，客户错误率 2.45%，上游错误率 6.60%。 +- `unidesk-codex-pool`：约 219 个请求、7 个客户错误，客户错误率 3.20%，上游错误率 13.24%，队列为 0。 + - `https://api.iceiu.com plus 0.03`（ID 33）：89.9/B/healthy，高置信；172 次尝试、3 次失败，失败率 1.74%，TTFT P95 42.87 秒；有 3 个 `forward_failed`，未形成客户错误。 + - `https://sub.yjxm1221.top plus 0.025`（ID 27）：15/E/poor，中置信；39 次尝试、26 次失败，失败率 66.67%，TTFT P95 202.12 秒；23 个切号请求，18 个恢复、5 个失败，3 次临时不可调度、3 个 `forward_failed`、7 个客户错误。它是该分组近期客户可见错误的主因。 + - `https://ai.whistlelads.com plus 0.03`（ID 28）：20/E/poor，低置信；11 次尝试全部失败；11 个切号请求全部由后续账号恢复，12 次临时不可调度，未形成客户错误。说明现有切号有效遮蔽了该账号故障，但源账号质量差。 + - `https://sub2.pokexiao.com plus 0.03`（ID 31）：样本内 100 分，但只有 8 次尝试，等级为 `insufficient`，不能据此认定为优质账号。 + - `https://api.lwylink.xyz pro 0.1`（ID 29）和 `https://api.lwylink.xyz plus 0.05`（ID 30）当前不可用且无近期成功证据。 +- `自用`：约 326 个请求、5 个客户错误，客户错误率 1.53%，上游错误率 2.15%，队列为 0。 + - `https://api.iceiu.com plus 0.03`（ID 33）：93.5/A/preferred，高置信；230 次尝试全部成功，TTFT P95 53.94 秒。 + - `https://sub2.pokexiao.com pro 0.05`（ID 32）：77.9/C/watch，中置信；33 次尝试、1 次失败，切号后恢复，TTFT P95 98.49 秒。 + - `lyon9801`（ID 15）：55.9/E/poor，中置信；34 次尝试、5 次失败，失败率 14.71%；5 次切号均未恢复，5 个客户错误，是该分组近期客户可见错误主因。 +- `grok` 分组近期无请求和错误。 +- 两个活跃分组均无排队，当前证据不支持把降低全局上游并发作为主修复方向。 +- 近期未观察到同账号重试事件；不能把“未观察到”解释为功能不存在。 + +## CLI 行为 + +- `runtime errors --all-groups`：所有分组概要和稳定分页。 +- `runtime errors --group `：分组内完整账号名称、评分、置信度、错误、切号和 TTFT 表格。 +- `runtime errors --group --account `：单账号详细归因和 request index。 +- 整组 `--json` 保留所有账号名称和质量概要；请求级证据通过 `--account` 下钻，不依赖手工 `limit`。 +- 单账号 JSON 实测 9248 字节、整组 JSON 实测 9058 字节，均低于 10240 字节输出预算；未出现 `outputTruncated`，也未暴露内部 `_requestIds`。 + +## 结论与后续建议 + +主因不是池总并发或排队，而是少数上游账号的高失败率和长 TTFT：`https://sub.yjxm1221.top plus 0.025` 与 `lyon9801` 会把错误直接传给客户，`https://ai.whistlelads.com plus 0.03` 虽被切号遮蔽但持续消耗 failover。下一步若获准调优，应优先基于高置信/中置信评分调整账号优先级或临时不可调度，并继续保持 `lyon9801` 的既有例外约束；低置信账号先积累样本，不应自动调度。任何自动评分到 runtime 的闭环都应另行审批，本任务没有实施。 + +## 验证 + +- `bun --check scripts/src/platform-infra-sub2api-codex/runtime.ts` +- `bun --check scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts` +- `bun --check scripts/src/output.ts` +- `bun test scripts/src/output.test.ts`：2 pass，0 fail。 +- `git diff --check` +- PK01 原生只读命令已验证 all-groups、group 2、group 3、account 27、默认表格和显式 JSON。 diff --git a/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.9_Task_Report.md b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.9_Task_Report.md new file mode 100644 index 00000000..6705c7d9 --- /dev/null +++ b/docs/MDTODO/details/sub2api-upstream-reliability/R2.2.9_Task_Report.md @@ -0,0 +1,48 @@ +# R2.2.9 任务报告 + +## 范围 + +只读检查 PK01 Sub2API 最近 30 分钟的所有分组、账号质量、客户可见错误、切号结果、临时不可调度和代表请求链路。未修改 runtime、YAML、账号调度或 Sub2API 运行面。 + +## 全局快照 + +抓取时间为 2026-07-14 06:14-06:15 UTC，窗口向前 30 分钟，数据会随滚动窗口变化。 + +- 所有分组共 440 个请求，7 个客户可见错误，客户错误率 1.59%；13 个上游错误，上游错误率 2.95%。 +- `unidesk-codex-pool`：约 257-258 个请求，4 个客户错误，错误率约 1.56%，上游错误率约 3.89%。 +- `自用`：约 182-183 个请求，3 个客户错误，错误率约 1.64%，上游错误率约 1.64%。 +- `grok`：无请求、无错误。 +- 两个活跃分组查询时并发约 8-9/218，排队为 0；当前证据不支持把降低池总并发作为主修复。 +- 当前有 7/9 个账号可用。`https://api.lwylink.xyz pro 0.1`（ID 29）和 `https://api.lwylink.xyz plus 0.05`（ID 30）因余额不足 403 处于错误状态。 + +## unidesk-codex-pool + +- `https://sub2.pokexiao.com plus 0.03`（ID 31）：95.6/A/preferred，高置信；84 次尝试全部成功，TTFT P95 39.95 秒。 +- `https://api.iceiu.com plus 0.03`（ID 33）：87.9/B/healthy，高置信；90 次尝试、2 次失败，失败率 2.22%，TTFT P95 46.90 秒；产生 2 个客户错误。 +- `https://sub.yjxm1221.top plus 0.025`（ID 27）：66.2/D/degraded，高置信；55 次尝试、6 次失败，失败率 10.91%，TTFT P95 17.56 秒；产生 2 个客户错误、4 个 `forward_failed`。其 2 次切号均恢复。 +- `https://ai.whistlelads.com plus 0.03`（ID 28）：7 次尝试全部失败，证据不足等级；7 次切号全部恢复、7 次临时不可调度，没有形成客户错误，说明内部 failover 有效遮蔽了该账号近期故障。 +- 该分组 4 个客户错误均为同步请求：2 个 400、2 个 404，模型为 `gpt-5.6-luna` 和 `gpt-5.6-sol`。 +- 分组共 8 次切号，8 次全部恢复；没有候选耗尽。 +- request `46bdd23b-e8bf-461d-bb3f-ad6313412ae3`：账号 `https://sub.yjxm1221.top plus 0.025`，最终 502；上游原文是 `400 Input must be a list`，总耗时约 977 秒。该错误更像请求结构或协议兼容错误，不应按临时上游过载盲目重试。 +- request `bac355f9-22fc-4d18-a899-8f609d847615`：账号 `https://sub.yjxm1221.top plus 0.025`，HTTP 200 后出现 `stream usage incomplete: missing terminal event`，trace 判为 degraded，说明成功状态不能代表客户收到完整流式响应。 + +## 自用 + +- `https://sub2.pokexiao.com plus 0.03`（ID 31）：99.1/A/preferred，中置信；23 次尝试全部成功，TTFT P95 15.98 秒。 +- `https://sub.yjxm1221.top plus 0.025`（ID 27）：98.5/A/preferred，高置信；55 次尝试全部成功，TTFT P95 19.94 秒。同一账号在不同分组的模型和请求构成不同，不能跨分组合并评分。 +- `https://api.iceiu.com plus 0.03`（ID 33）：86.7/B/healthy，中置信；39 次尝试、1 次失败，产生 1 个 524 客户错误；1 次切号失败。 +- `lyon9801`（ID 15）：81.6/B/healthy，中置信；49 次尝试、3 次失败，失败率 6.12%，TTFT P95 9.89 秒；产生 2 个 503 流式客户错误。虽然综合评分为 B，但它仍是该分组近期客户错误主因。 +- 分组共 3 次切号，全部失败并出现 3 次选号失败。 +- request `da415526-b124-4e23-a4a7-ea48154c851e`：账号 `lyon9801`，流式 `gpt-5.6-sol`；约 1556 秒后才收到 503 并触发切号，此时 context 已取消，选号在约 1 毫秒内失败，trace 判为 `failover-attempted-no-candidate`。这是 failover 时机过晚，不是池容量不足。 +- request `4bba7f9b-15e3-4891-b063-bd65fb6e3704`：账号 `lyon9801`，HTTP 200 后发生 `upstream response failed`，trace 判为 degraded，客户可能收到不完整流。 + +## 判断 + +- 相比上一轮 60 分钟快照，当前短窗口的客户错误率和上游错误率数值更低，但窗口不同且持续滚动，不能直接解释为配置调优效果。 +- 当前主要客户可见错误已经从纯上游过载，分化为三类： + - 请求/协议兼容错误：`Input must be a list`、404； + - 超长等待后才 failover，客户上下文已取消； + - HTTP 200 已提交后缺少终止事件或后续流式失败。 +- 对第一类不应无差别重试；对第二类应继续调查 response-header/首事件等待和 failover 触发时机；对第三类需要把 degraded 流式结果纳入客户错误治理，不能仅看 HTTP 状态。 +- 最近 30 分钟未观察到同账号重试事件；不能把零事件解释为重试能力不存在。 +- 当前只完成分析，没有下发任何调优。 diff --git a/docs/MDTODO/sub2api-upstream-reliability.md b/docs/MDTODO/sub2api-upstream-reliability.md index 3a3c36ba..27bc7422 100644 --- a/docs/MDTODO/sub2api-upstream-reliability.md +++ b/docs/MDTODO/sub2api-upstream-reliability.md @@ -41,6 +41,60 @@ #### R2.2.7 按仓库 P0 拆分超过 3000 行的 `remote-python-sync.ts`：把大段内嵌 Python 按 trace、sync/validate 与 sentinel 职责迁入原生模块，单文件收敛到 2000 行以下，并保持受控 CLI 行为与只读诊断契约不变，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.7_Task_Report.md)。 +#### R2.2.8 [completed] + +增强 Sub2API CLI 的上游账号粒度质量分析与自动评分：使用原生 Ops 归因账号请求量、客户/上游错误、触发切号、切号结果、同账号重试、临时不可调度和首字延迟覆盖率/分位数，输出可解释评分、置信度和扣分原因；只做分析，不自动修改账号 runtime 或调度配置，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.8_Task_Report.md)。 +#### R2.2.9 [completed] + +重新抓取 PK01 Sub2API 最近 30 分钟所有分组的客户可见错误、上游错误、账号质量、切号恢复、临时不可调度和首字延迟，按完整账号名称报告主因；只做原生只读分析，不修改 runtime、YAML 或调度配置，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.9_Task_Report.md)。 +#### R2.2.10 [completed] + +只读专项调查 lyon9801 的近期 503 流式错误：增强 Sub2API CLI 同时采集账号 proxy 绑定、容器环境代理摘要、DNS/路由、连接与出网基础设施证据，结合真实 request trace 和源码链路区分 OpenAI 上游、PK01 网络、代理、超时或 Sub2API 转发问题；不发探针、不修改 runtime、YAML、代理或调度配置，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.10_Task_Report.md)。 +#### R2.2.11 [completed] + +只读复取并综合分析最近2小时所有 Sub2API 分组的客户可见错误、上游错误、账号质量、failover、候选耗尽和流式 degraded 情况，报告完整账号名称，不发探针且不修改 runtime、YAML、代理或调度配置，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.11_Task_Report.md)。 +#### R2.2.12 [completed] + +快速缓解账号 https://sub.yjxm1221.top plus 0.025 的 Input must be a list 客户可见400：在 owning YAML 模板加入精确400短语并保留该 runtime-manual 账号现有规则，通过精准 runtime CRUD dry-run、确认下发和短窗口只读复取验证当前请求能够冷却并切号，不泛化匹配其他400，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.12_Task_Report.md)。 +#### R2.2.13 [completed] + +向当前PK01全部非 lyon9801 OpenAI账号精准批量下发 codex-upstream-failover 统一临时不可调度模板：从runtime账号清单固定显式selector，排除lyon9801，先dry-run再confirm并逐账号对账，不修改分组、优先级、容量、proxy或凭据，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.13_Task_Report.md)。 +#### R2.2.14 [in_progress] + +由 Artificer 实现 Sub2API runtime 精准批量 CRUD 的命令内自动回读对账：confirm 后批量回读全部selector并输出write result、actual、reconciled、mismatch和部分写入摘要，避免外部逐账号get；关联 [GitHub issue #1998](https://github.com/pikasTech/unidesk/issues/1998)，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.14_Task_Report.md)。 +#### R2.2.15 [completed] + +继续只读分析统一非 auth 上游模板与超时预算的下一轮调优方案：交由独立 Artificer 调查 [Issue #1999](https://github.com/pikasTech/unidesk/issues/1999)，基于下发后真实400切号成功、旧长请求context canceled、其他invalid input id与HTTP 200流式degraded证据，区分可安全泛化的精确模板、需要Sub2API源码能力和不应自动处理的客户参数错误，不继续盲目扩规则，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.15_Task_Report.md)。 + +#### R2.2.16 [completed] + +紧急只读追踪用户 mail0795336304@163.com 反馈请求卡住15分钟：定位用户ID、近30分钟请求、当前长请求/并发、request ID、账号与模型，区分排队、上游首字、流式间隔、代理网络或客户连接，不取消请求且不修改runtime、YAML、proxy或调度，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.16_Task_Report.md)。 +#### R2.2.17 [in_progress] + +复盘用户直接反馈的只读追踪过程并交由 Artificer 实现 [Issue #2000](https://github.com/pikasTech/unidesk/issues/2000)：增加通用的一键用户追踪 CLI，以邮箱或用户ID关联请求空窗、用户/API Key并发、client/internal request ID、上游账号名称、proxy基础设施和证据化归因，采用Kubernetes列表风格与稳定ID渐进披露，减少手工API与重复trace调用，不改运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.17_Task_Report.md)。 +#### R2.2.18 [completed] + +将 PK01 的 Sub2API 通过 owning YAML 从 v0.1.153 受控升级到 v0.1.155：只修改 PK01 target镜像版本，依次执行状态核对、plan、apply dry-run、镜像预拉、受控apply、状态与公开健康验证，保持账号池runtime和Codex pool配置原样，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.18_Task_Report.md)。 +#### R2.2.19 [in_progress] + +交由 Artificer 实现 [Issue #2004](https://github.com/pikasTech/unidesk/issues/2004)：增强 Sub2API CLI 拉取 v0.1.155 原生智能诊断与渠道监控状态，泛化所有分组/渠道概要和指定对象下钻，以Kubernetes列表风格呈现TTFT、上游与客户错误率、原生建议和证据边界，以及7/15/30天渠道运行状态、平台/模型、对话延迟、端点PING、可用率、最近记录和刷新事实，使用稳定ID渐进披露，不重算第二套诊断且不修改运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.19_Task_Report.md)。 + +#### R2.2.20 [completed] + +快速缓解错误 #17440：基于 request e5c08dde-9e60-4a14-9458-5574cec00b65 的真实链路，将精确短语 Input must be a list 补入通用 504 临时不可调度规则，对全部非 lyon9801 账号精准批量 dry-run、确认下发和自动回读对账，不改容量、优先级、代理或 Sub2API 源码，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.20_Task_Report.md)。 +#### R2.2.21 [in_progress] + +交由 Artificer 实现 [Issue #2013](https://github.com/pikasTech/unidesk/issues/2013)：定义并实现 Sub2API P0/P1/P2 故障快速查询 CLI，P0 为上游故障穿透客户，P1 为首字延迟偏高且普通与 compact 分开统计并突出极端离群，P2 为上游错误率偏高；支持所有分组概要与按等级、分组、账号名称、模型、请求类型、稳定 request ID 下钻，默认 Kubernetes 列表风格和固定分页，不靠手工 limit，不修改运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.21_Task_Report.md)。 + +#### R2.2.22 [completed] + +主代理只读追踪客户错误 #17490（request ID 4eb9a84b-7346-47a8-bb66-fb7b44317279）：使用 Sub2API 原生 Ops、trace、账号完整名称、failover/retry/临时不可调度和近期窗口证据，定位账号 https://sub2.pokexiao.com plus 0.03 的同步502为何穿透客户，区分候选耗尽、context取消、不可重试阶段或模板未命中；先调查，不盲目新增generic 502规则，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.22_Task_Report.md)。 +#### R2.2.23 [in_progress] + +交由 Artificer 实现 [Issue #2016](https://github.com/pikasTech/unidesk/issues/2016)：增强 CLI 对 Sub2API 内部账号/渠道监控的原生只读访问，展示探测 timeout、cadence、阶段耗时、账号选择、上游首字节、切号/重试和最终状态，支持所有监控对象概要与稳定 ID 下钻，默认 Kubernetes 列表风格且不靠手工 limit；不修改运行面、不发新探针，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.23_Task_Report.md)。 +#### R2.2.24 [completed] + +只读调查错误 #17511（request ID `f7ff0efe-64d3-4dd7-8716-20bdfc3a2f90`）为何上游 502 后未成功切号，重建流式请求的账号选择、首字节、响应提交、failover、排除列表、context deadline 与最终客户错误链，并与内部 monitor 请求 #17490 对照，不修改运行面，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.2.24_Task_Report.md)。 ### R2.3 调查并收敛 Sub2API 原生 system-log 索引未保留策略事件的能力缺口，完成任务后将详细报告写入[任务报告](./details/sub2api-upstream-reliability/R2.3_Task_Report.md)。 diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-options.ts b/scripts/src/platform-infra-sub2api-codex/runtime-options.ts new file mode 100644 index 00000000..da45d5dd --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/runtime-options.ts @@ -0,0 +1,120 @@ +import { defaultCodexPoolRuntimeTargetId } from "./runtime-target"; + +export type RuntimeAction = "list" | "get" | "errors" | "infrastructure" | "apply" | "delete"; + +export interface RuntimeOptions { + action: RuntimeAction; + account: string | null; + accounts: string[]; + group: string | null; + allGroups: boolean; + platform: string | null; + pageToken: string | null; + template: string | null; + kind: "temp-unschedulable"; + confirm: boolean; + full: boolean; + raw: boolean; + json: boolean; + since: string; + tail: number; + targetId: string; +} + +export function parseRuntimeOptions(args: string[]): RuntimeOptions { + const [actionRaw = "list", ...rest] = args; + if (!(["list", "get", "errors", "infrastructure", "apply", "delete"] as string[]).includes(actionRaw)) { + throw new Error("runtime usage: list|get|errors|infrastructure|apply|delete [--account |--accounts ] [--group |--all-groups] [--platform ] [--page-token ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--json|--full|--raw]"); + } + let account: string | null = null; + let accounts: string[] = []; + let group: string | null = null; + let allGroups = false; + let platform: string | null = null; + let pageToken: string | null = null; + let template: string | null = null; + let kind = "temp-unschedulable" as const; + let confirm = false; + let full = false; + let raw = false; + let json = false; + let since = "24h"; + let tail = 50_000; + let targetId = defaultCodexPoolRuntimeTargetId(); + for (let index = 0; index < rest.length; index += 1) { + const arg = rest[index]!; + const readValue = (name: string): string => { + const value = rest[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); + index += 1; + return value.trim(); + }; + if (arg === "--confirm") confirm = true; + else if (arg === "--full") full = true; + else if (arg === "--raw") raw = true; + else if (arg === "--json") json = true; + else if (arg === "-o" && readValue("-o") === "json") json = true; + else if (arg === "--account") account = readValue("--account"); + else if (arg.startsWith("--account=")) account = arg.slice("--account=".length).trim(); + else if (arg === "--accounts") accounts = parseAccountSelectors(readValue("--accounts")); + else if (arg.startsWith("--accounts=")) accounts = parseAccountSelectors(arg.slice("--accounts=".length)); + else if (arg === "--group") group = readValue("--group"); + else if (arg.startsWith("--group=")) group = arg.slice("--group=".length).trim(); + else if (arg === "--all-groups") allGroups = true; + else if (arg === "--platform") platform = readValue("--platform"); + else if (arg.startsWith("--platform=")) platform = arg.slice("--platform=".length).trim(); + else if (arg === "--page-token") pageToken = readValue("--page-token"); + else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim(); + else if (arg === "--template") template = readValue("--template"); + else if (arg.startsWith("--template=")) template = arg.slice("--template=".length).trim(); + else if (arg === "--kind") kind = parseRuntimeKind(readValue("--kind")); + else if (arg.startsWith("--kind=")) kind = parseRuntimeKind(arg.slice("--kind=".length)); + else if (arg === "--target") targetId = readValue("--target"); + else if (arg.startsWith("--target=")) targetId = arg.slice("--target=".length).trim(); + else if (arg === "--since") since = readValue("--since"); + else if (arg.startsWith("--since=")) since = arg.slice("--since=".length).trim(); + else if (arg === "--tail") tail = parseRuntimeTail(readValue("--tail")); + else if (arg.startsWith("--tail=")) tail = parseRuntimeTail(arg.slice("--tail=".length)); + else throw new Error(`unsupported runtime option: ${arg}`); + } + const action = actionRaw as RuntimeAction; + if (account !== null && accounts.length > 0) throw new Error("use only one of --account or --accounts"); + if ((action === "get" || action === "infrastructure") && !account) throw new Error(`runtime ${action} requires --account `); + if ((action === "apply" || action === "delete") && account === null && accounts.length === 0) throw new Error(`runtime ${action} requires --account or --accounts`); + if (accounts.length > 0 && action !== "apply" && action !== "delete") throw new Error(`runtime ${action} does not accept --accounts`); + if (account !== null && (account.length > 256 || /[\r\n]/u.test(account))) throw new Error("--account has an unsupported format"); + if (group !== null && (group.length > 256 || /[\r\n]/u.test(group))) throw new Error("--group has an unsupported format"); + if (platform !== null && (!/^[A-Za-z0-9._-]+$/u.test(platform) || platform.length > 64)) throw new Error("--platform has an unsupported format"); + if (pageToken !== null && (!/^\d+$/u.test(pageToken) || pageToken.length > 20)) throw new Error("--page-token must be a numeric group cursor"); + if (group !== null && allGroups) throw new Error("use only one of --group or --all-groups"); + if ((allGroups || pageToken !== null) && action !== "errors") throw new Error(`runtime ${action} does not accept all-groups pagination options`); + if ((group !== null || platform !== null) && action !== "errors" && action !== "infrastructure") throw new Error(`runtime ${action} does not accept group scope options`); + if (pageToken !== null && !allGroups) throw new Error("--page-token requires --all-groups"); + if (account !== null && allGroups) throw new Error("--account cannot be combined with --all-groups; use --group first"); + if (action === "apply" && !template) throw new Error("runtime apply requires --template "); + if (action !== "apply" && template !== null) throw new Error(`runtime ${action} does not accept --template`); + if (action !== "errors" && action !== "infrastructure" && (since !== "24h" || tail !== 50_000)) throw new Error(`runtime ${action} does not accept --since or --tail`); + if (!/^\d+[mhd]$/u.test(since)) throw new Error("--since must use m, h, or d"); + if (["list", "get", "errors", "infrastructure"].includes(action) && confirm) throw new Error(`runtime ${action} does not accept --confirm`); + if ([full, raw, json].filter(Boolean).length > 1) throw new Error("use only one of --json, --full, or --raw"); + return { action, account, accounts, group, allGroups, platform, pageToken, template, kind, confirm, full, raw, json, since, tail, targetId }; +} + +function parseAccountSelectors(value: string): string[] { + const selectors = value.split(",").map((item) => item.trim()).filter(Boolean); + if (selectors.length === 0 || selectors.length > 100) throw new Error("--accounts requires 1 to 100 comma-separated selectors"); + if (selectors.some((item) => item.length > 256 || /[\r\n]/u.test(item))) throw new Error("--accounts has an unsupported selector"); + if (new Set(selectors).size !== selectors.length) throw new Error("--accounts contains duplicate selectors"); + return selectors; +} + +function parseRuntimeKind(value: string): "temp-unschedulable" { + if (value !== "temp-unschedulable") throw new Error("--kind must be temp-unschedulable"); + return value; +} + +function parseRuntimeTail(value: string): number { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 100 || parsed > 200_000) throw new Error("--tail must be an integer from 100 to 200000"); + return parsed; +} diff --git a/scripts/src/platform-infra-sub2api-codex/runtime.ts b/scripts/src/platform-infra-sub2api-codex/runtime.ts index 077d1661..88ed5dcf 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime.ts @@ -7,27 +7,8 @@ import { protectedManualAccountsForTarget } from "./public-exposure"; import { renderSub2ApiTempUnschedulableCredentials } from "./redaction"; import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote"; import { renderedCliResult, renderTable } from "./render"; -import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target"; - -type RuntimeAction = "list" | "get" | "errors" | "apply" | "delete"; - -interface RuntimeOptions { - action: RuntimeAction; - account: string | null; - group: string | null; - allGroups: boolean; - platform: string | null; - pageToken: string | null; - template: string | null; - kind: "temp-unschedulable"; - confirm: boolean; - full: boolean; - raw: boolean; - json: boolean; - since: string; - tail: number; - targetId: string; -} +import { parseRuntimeOptions, type RuntimeOptions } from "./runtime-options"; +import { codexPoolRuntimeTarget } from "./runtime-target"; export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { const options = parseRuntimeOptions(args); @@ -40,6 +21,7 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P const payload = { action: options.action, account: options.account, + accounts: options.accounts, group: options.group, allGroups: options.allGroups, platform: options.platform, @@ -90,6 +72,7 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P request: { operation: options.action, account: options.account, + accounts: options.accounts, group: options.group, allGroups: options.allGroups, platform: options.platform, @@ -97,8 +80,8 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P template: options.template, kind: options.kind, confirm: options.confirm, - since: options.action === "errors" ? options.since : undefined, - tail: options.action === "errors" ? options.tail : undefined, + since: options.action === "errors" || options.action === "infrastructure" ? options.since : undefined, + tail: options.action === "errors" || options.action === "infrastructure" ? options.tail : undefined, }, runtime: compactRuntimeErrors(parsed, options), remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }), @@ -159,6 +142,54 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; const groupFlag = options.group === null ? "" : ` --group ${options.group}`; const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; + if (options.json && options.account === null) { + return { + ok: runtime.ok, + operation: runtime.operation, + mutation: runtime.mutation, + group: runtime.group, + endpoint: runtime.endpoint, + errors: { + window: errors.window, + nativeOps: { + source: nativeOps.source, + overview: nativeOps.overview, + accountQuality: compactAccountQuality(nativeOps.accountQuality, false), + accountAvailability: { + status: availability.status, + totalAccounts: availabilityGroup.total_accounts, + availableCount: availabilityGroup.available_count, + rateLimitCount: availabilityGroup.rate_limit_count, + errorCount: availabilityGroup.error_count, + unavailableAccounts: Array.isArray(availability.unavailableAccounts) + ? availability.unavailableAccounts.map((item) => { + const account = runtimeRecord(item) ?? {}; + return { + accountId: account.accountId, + accountName: account.accountName, + status: account.status, + reason: account.error, + }; + }) + : [], + }, + concurrency: { + status: concurrency.status, + currentInUse: concurrencyGroup.current_in_use, + maxCapacity: concurrencyGroup.max_capacity, + loadPercentage: concurrencyGroup.load_percentage, + waitingInQueue: concurrencyGroup.waiting_in_queue, + }, + }, + }, + disclosure: { + account: `${baseCommand}${groupFlag}${platformFlag} --account `, + trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", + note: "This group overview keeps every account name and quality summary. Use account drilldown for request indexes and complete failure evidence.", + }, + valuesPrinted: false, + }; + } return { ok: runtime.ok, operation: runtime.operation, @@ -170,9 +201,10 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown nativeOps: { source: nativeOps.source, overview: nativeOps.overview, - responseHeaderTimeout: nativeOps.responseHeaderTimeout, - requestTiming: nativeOps.requestTiming, - customerErrors: nativeOps.customerErrors, + responseHeaderTimeout: options.account === null || !options.json ? nativeOps.responseHeaderTimeout : undefined, + requestTiming: options.account === null || !options.json ? compactRequestTiming(nativeOps.requestTiming) : undefined, + customerErrors: compactCustomerErrors(nativeOps.customerErrors, options.account !== null || !options.json), + accountQuality: compactAccountQuality(nativeOps.accountQuality, options.account !== null || !options.json), accountAvailability: { status: availability.status, totalAccounts: availabilityGroup.total_accounts, @@ -215,15 +247,11 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown failover: effectiveness.failover, forwardFailure: effectiveness.forwardFailure, selectFailedEvents: effectiveness.selectFailedEvents, - requestIdCoverage: effectiveness.requestIdCoverage, - gatewayOutcomeEvidence: effectiveness.gatewayOutcomeEvidence, requestIndexes, sourceStatus: { status: sourceStatus.status, - source: sourceStatus.source, fallbackUsed: sourceStatus.fallbackUsed, fallbackReason: sourceStatus.fallbackReason, - fallbackEventCount: sourceStatus.fallbackEventCount, }, }, }, @@ -237,6 +265,99 @@ function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown }; } +function compactRequestTiming(value: unknown): unknown { + const timing = runtimeRecord(value); + if (timing === null) return value; + return { + status: timing.status, + reason: timing.reason, + source: timing.source, + ttft: timing.ttft, + duration: timing.duration, + observedFloorSeconds: timing.observedFloorSeconds, + attribution: timing.attribution, + }; +} + +function compactCustomerErrors(value: unknown, detailed: boolean): unknown { + const profile = runtimeRecord(value); + if (profile === null) return value; + return { + status: profile.status, + reason: profile.reason, + source: profile.source, + total: profile.total, + modelBuckets: runtimeRecords(profile.modelBuckets).slice(0, 5), + accountBuckets: detailed ? runtimeRecords(profile.accountBuckets).slice(0, 5) : undefined, + statusBuckets: runtimeRecords(profile.statusBuckets).slice(0, 5), + streamBuckets: runtimeRecords(profile.streamBuckets).slice(0, 5), + }; +} + +function compactAccountQuality(value: unknown, detailed: boolean): unknown { + const quality = runtimeRecord(value); + if (quality === null) return value; + return { + source: quality.source, + scope: quality.scope, + window: quality.window, + scoreRule: (() => { + const rule = runtimeRecord(quality.scoreRule) ?? {}; + return detailed ? { + reliabilityWeight: rule.reliabilityWeight, + latencyWeight: rule.latencyWeight, + availabilityWeight: rule.availabilityWeight, + failureDefinition: rule.failureDefinition, + missingMetric: rule.missingMetric, + } : { + reliabilityWeight: rule.reliabilityWeight, + latencyWeight: rule.latencyWeight, + availabilityWeight: rule.availabilityWeight, + }; + })(), + accounts: runtimeRecords(quality.accounts).map((account) => detailed ? { + accountId: account.accountId, + accountName: account.accountName, + status: account.status, + currentlyAvailable: account.currentlyAvailable, + score: account.score, + grade: account.grade, + assessment: account.assessment, + confidence: account.confidence, + observedAttempts: account.observedAttempts, + successRequests: account.successRequests, + failureRequests: account.failureRequests, + failureRate: account.failureRate, + firstTokenSamples: account.firstTokenSamples, + firstTokenCoverage: account.firstTokenCoverage, + ttftP95Ms: account.ttftP95Ms, + customerErrorRequests: account.customerErrorRequests, + failoverRequests: account.failoverRequests, + failoverRecovered: account.failoverRecovered, + failoverFailed: account.failoverFailed, + sameAccountRetryEvents: account.sameAccountRetryEvents, + tempUnschedulableEvents: account.tempUnschedulableEvents, + forwardFailedRequests: account.forwardFailedRequests, + reasons: account.reasons, + } : { + accountId: account.accountId, + accountName: account.accountName, + currentlyAvailable: account.currentlyAvailable, + score: account.score, + grade: account.grade, + confidence: account.confidence, + observedAttempts: account.observedAttempts, + failureRate: account.failureRate, + ttftP95Ms: account.ttftP95Ms, + customerErrorRequests: account.customerErrorRequests, + failoverRequests: account.failoverRequests, + failoverRecovered: account.failoverRecovered, + failoverFailed: account.failoverFailed, + }), + attribution: detailed ? quality.attribution : undefined, + }; +} + function runtimeRecord(value: unknown): Record | null { return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : null; } @@ -269,6 +390,7 @@ function renderRuntimeResult(response: Record, options: Runtime const ok = response.ok === true; const lines: string[] = []; if (options.action === "errors") renderRuntimeErrors(lines, runtime, options); + else if (options.action === "infrastructure") renderRuntimeInfrastructure(lines, runtime); else if (options.action === "list") renderRuntimeAccountList(lines, runtime); else if (options.action === "get") renderRuntimeAccountDetail(lines, runtime); else renderRuntimeMutation(lines, runtime, options); @@ -327,6 +449,7 @@ function renderRuntimeErrors(lines: string[], runtime: Record, const ttft = runtimeRecord(timing.ttft) ?? {}; const duration = runtimeRecord(timing.duration) ?? {}; const customerErrors = runtimeRecord(nativeOps.customerErrors) ?? {}; + const accountQuality = runtimeRecord(nativeOps.accountQuality) ?? {}; lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`); lines.push(renderTable([ ["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"], @@ -358,6 +481,32 @@ function renderRuntimeErrors(lines: string[], runtime: Record, ["MODEL", "REQUESTS", "FIRST_TOKEN_ROWS", "COVERAGE", "AVG_TTFT_MS", "AVG_DURATION_MS", "AVG_TOKENS_S"], ...modelTiming.map((item) => [runtimeShort(item.model, 32), runtimeText(item.requestCount), runtimeText(item.requestsWithFirstToken), runtimeRate(item.firstTokenCoverage), runtimeText(item.avgFirstTokenMs), runtimeText(item.avgDurationMs), runtimeText(item.avgTokensPerSec)]), ])); + const qualityAccounts = runtimeRecords(accountQuality.accounts); + if (qualityAccounts.length > 0) { + lines.push("", "ACCOUNT QUALITY", renderTable([ + ["ACCOUNT", "ID", "SCORE", "GRADE", "QUALITY", "CONF", "ATTEMPTS", "SUCCESS", "FAIL", "FAIL_RATE", "TTFT_P95_MS", "TTFT_N", "AVAILABLE", "REASONS"], + ...qualityAccounts.map((item) => [ + runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.score), runtimeText(item.grade), runtimeText(item.assessment), runtimeText(item.confidence), + runtimeText(item.observedAttempts), runtimeText(item.successRequests), runtimeText(item.failureRequests), runtimeRate(item.failureRate), + runtimeText(item.ttftP95Ms), runtimeText(item.firstTokenSamples), runtimeText(item.currentlyAvailable), + runtimeShort(Array.isArray(item.reasons) ? item.reasons.join(",") : "", 58), + ]), + ])); + const controlRows = qualityAccounts.filter((item) => [ + item.failoverRequests, item.sameAccountRetryEvents, item.tempUnschedulableEvents, item.forwardFailedRequests, item.customerErrorRequests, + ].some((value) => typeof value === "number" && value > 0)); + if (controlRows.length > 0) lines.push("ACCOUNT FAILURE CONTROL", renderTable([ + ["ACCOUNT", "ID", "FAILOVER", "RECOVERED", "FAILED", "MISSING", "RETRY", "TEMP_UNSCHED", "FORWARD_FAIL", "CUSTOMER_ERR"], + ...controlRows.map((item) => [ + runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.failoverRequests), runtimeText(item.failoverRecovered), + runtimeText(item.failoverFailed), runtimeText(item.failoverOutcomeMissing), runtimeText(item.sameAccountRetryEvents), + runtimeText(item.tempUnschedulableEvents), runtimeText(item.forwardFailedRequests), runtimeText(item.customerErrorRequests), + ]), + ])); + const scoreRule = runtimeRecord(accountQuality.scoreRule) ?? {}; + lines.push(`SCORE RULE reliability=${runtimeText(scoreRule.reliabilityWeight)} latency=${runtimeText(scoreRule.latencyWeight)} availability=${runtimeText(scoreRule.availabilityWeight)} confidence=separate`); + if (typeof accountQuality.attribution === "string") lines.push(`NOTE: ${accountQuality.attribution}`); + } const errorModels = runtimeRecords(customerErrors.modelBuckets); const errorAccounts = runtimeRecords(customerErrors.accountBuckets); const errorDimensions = [ @@ -462,7 +611,100 @@ function renderRuntimeAccountDetail(lines: string[], runtime: Record): void { + const account = runtimeRecord(runtime.account) ?? {}; + const infra = runtimeRecord(runtime.infrastructure) ?? {}; + const proxy = runtimeRecord(infra.accountProxy) ?? {}; + const app = runtimeRecord(infra.appRuntime) ?? {}; + const route = runtimeRecord(infra.defaultRoute) ?? {}; + const dns = runtimeRecord(infra.dns) ?? {}; + const sockets = runtimeRecord(infra.proxySockets) ?? {}; + const listener = runtimeRecord(infra.proxyEndpointOwner) ?? {}; + const serviceLogs = runtimeRecord(infra.proxyServiceEvidence) ?? {}; + const logs = runtimeRecord(infra.recentNetworkEvidence) ?? {}; + lines.push(`SUB2API RUNTIME INFRASTRUCTURE ok=${runtime.ok === true ? "true" : "false"} since=${runtimeText(logs.window)}`); + if (runtime.ok !== true && typeof runtime.error === "string") lines.push(`ERROR: ${runtime.error}`); + lines.push(renderTable([ + ["ACCOUNT", "ID", "TYPE", "STATUS", "SCHED", "PROXY_ID", "PROXY_NAME", "PROTOCOL", "PROXY_ENDPOINT", "PROXY_STATUS"], + [ + runtimeText(account.accountName), runtimeText(account.accountId), runtimeText(account.type), runtimeText(account.status), + runtimeText(account.schedulable), runtimeText(proxy.id), runtimeText(proxy.name), runtimeText(proxy.protocol), + `${runtimeText(proxy.host)}:${runtimeText(proxy.port)}`, runtimeText(proxy.status), + ], + ])); + lines.push("", "APP RUNTIME", renderTable([ + ["MODE", "CONTAINER", "STATE", "RESTARTS", "OOM", "NETWORK_MODE", "STARTED_AT"], + [ + runtimeText(app.runtimeMode), runtimeText(app.name), runtimeText(app.state), runtimeText(app.restartCount), + runtimeText(app.oomKilled), runtimeText(app.networkMode), runtimeText(app.startedAt), + ], + ])); + const envRows = runtimeRecords(infra.proxyEnvironment); + if (envRows.length > 0) lines.push("PROXY ENVIRONMENT", renderTable([ + ["VARIABLE", "PRESENT", "SCHEME", "HOST", "PORT", "CREDENTIALS_REDACTED"], + ...envRows.map((item) => [runtimeText(item.name), runtimeText(item.present), runtimeText(item.scheme), runtimeText(item.host), runtimeText(item.port), runtimeText(item.credentialsRedacted)]), + ])); + lines.push("NETWORK", renderTable([ + ["DEFAULT_IFACE", "DEFAULT_GATEWAY", "DNS_NAMESERVERS", "DNS_SEARCH", "PROXY_RESOLVED_IPS", "SOCKETS", "ESTABLISHED"], + [ + runtimeText(route.interface), runtimeText(route.gateway), runtimeText(dns.nameservers), runtimeText(dns.search), + runtimeText(sockets.resolvedIps), runtimeText(sockets.total), runtimeText(sockets.established), + ], + ])); + const listeners = runtimeRecords(listener.listeners); + if (listeners.length > 0) lines.push("PROXY ENDPOINT OWNER", renderTable([ + ["LISTEN", "PROCESS", "PID", "UNIT", "ACTIVE", "RESTARTS", "TCP_PEERS", "UDP_PEERS"], + ...listeners.map((item) => [runtimeText(item.localEndpoint), runtimeText(item.processName), runtimeText(item.pid), runtimeText(item.systemdUnit), runtimeText(item.activeState), runtimeText(item.serviceRestarts), runtimeText(item.establishedPeers), runtimeText(item.udpPeers)]), + ])); + const serviceBuckets = runtimeRecords(serviceLogs.buckets); + if (serviceBuckets.length > 0) lines.push("PROXY SERVICE EVIDENCE", renderTable([ + ["CAUSE", "COUNT"], + ...serviceBuckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), + ])); + const serviceSamples = runtimeRecords(serviceLogs.samples); + if (serviceSamples.length > 0) lines.push("PROXY SERVICE INDEX", renderTable([ + ["AT", "CAUSE", "MESSAGE"], + ...serviceSamples.map((item) => [runtimeText(item.at), runtimeText(item.cause), runtimeShort(item.message, 100)]), + ])); + const interfaces = runtimeRecords(infra.interfaces); + if (interfaces.length > 0) lines.push("INTERFACES", renderTable([ + ["NAME", "RX_BYTES", "TX_BYTES", "RX_ERRORS", "TX_ERRORS", "RX_DROPPED", "TX_DROPPED"], + ...interfaces.map((item) => [runtimeText(item.name), runtimeText(item.rxBytes), runtimeText(item.txBytes), runtimeText(item.rxErrors), runtimeText(item.txErrors), runtimeText(item.rxDropped), runtimeText(item.txDropped)]), + ])); + const buckets = runtimeRecords(logs.buckets); + lines.push("", `RECENT NETWORK EVIDENCE matched=${runtimeText(logs.matchedEvents)} source=${runtimeText(logs.source)}`); + if (buckets.length > 0) lines.push(renderTable([ + ["CAUSE", "COUNT"], + ...buckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), + ])); + const samples = runtimeRecords(logs.samples); + if (samples.length > 0) lines.push("EVIDENCE INDEX", renderTable([ + ["AT", "REQUEST_ID", "CAUSE", "STATUS", "MESSAGE"], + ...samples.map((item) => [runtimeText(item.at), runtimeText(item.requestId), runtimeText(item.cause), runtimeText(item.statusCode), runtimeShort(item.message, 90)]), + ])); + if (typeof infra.attribution === "string") lines.push(`NOTE: ${infra.attribution}`); +} + function renderRuntimeMutation(lines: string[], runtime: Record, options: RuntimeOptions): void { + const plans = runtimeRecords(runtime.plans); + if (plans.length > 0) { + const summary = runtimeRecord(runtime.summary) ?? {}; + lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); + lines.push(renderTable([ + ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES", "RESULT"], + ...plans.map((plan) => { + const account = runtimeRecord(plan.account) ?? {}; + const before = runtimeRecord(plan.before) ?? {}; + const after = runtimeRecord(plan.after) ?? {}; + return [ + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), + runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), runtimeText(plan.result), + ]; + }), + ])); + lines.push(`SUMMARY selected=${runtimeText(summary.selected)} changed=${runtimeText(summary.changed)} succeeded=${runtimeText(summary.succeeded)} failed=${runtimeText(summary.failed)}`); + return; + } const plan = runtimeRecord(runtime.plan) ?? {}; const account = runtimeRecord(plan.account) ?? runtimeRecord(runtime.account) ?? {}; const before = runtimeRecord(plan.before) ?? {}; @@ -477,90 +719,6 @@ function renderRuntimeMutation(lines: string[], runtime: Record ])); } -export function parseRuntimeOptions(args: string[]): RuntimeOptions { - const [actionRaw = "list", ...rest] = args; - if (actionRaw !== "list" && actionRaw !== "get" && actionRaw !== "errors" && actionRaw !== "apply" && actionRaw !== "delete") { - throw new Error("runtime usage: list|get|errors|apply|delete [--account ] [--group |--all-groups] [--platform ] [--page-token ] [--since 24h] [--tail 50000] [--template ] [--kind temp-unschedulable] [--confirm] [--target ] [--json|--full|--raw]"); - } - let account: string | null = null; - let group: string | null = null; - let allGroups = false; - let platform: string | null = null; - let pageToken: string | null = null; - let template: string | null = null; - let kind = "temp-unschedulable" as const; - let confirm = false; - let full = false; - let raw = false; - let json = false; - let since = "24h"; - let tail = 50_000; - let targetId = defaultCodexPoolRuntimeTargetId(); - for (let index = 0; index < rest.length; index += 1) { - const arg = rest[index]!; - const readValue = (name: string): string => { - const value = rest[index + 1]; - if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); - index += 1; - return value.trim(); - }; - if (arg === "--confirm") confirm = true; - else if (arg === "--full") full = true; - else if (arg === "--raw") raw = true; - else if (arg === "--json") json = true; - else if (arg === "-o" && readValue("-o") === "json") json = true; - else if (arg === "--account") account = readValue("--account"); - else if (arg.startsWith("--account=")) account = arg.slice("--account=".length).trim(); - else if (arg === "--group") group = readValue("--group"); - else if (arg.startsWith("--group=")) group = arg.slice("--group=".length).trim(); - else if (arg === "--all-groups") allGroups = true; - else if (arg === "--platform") platform = readValue("--platform"); - else if (arg.startsWith("--platform=")) platform = arg.slice("--platform=".length).trim(); - else if (arg === "--page-token") pageToken = readValue("--page-token"); - else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim(); - else if (arg === "--template") template = readValue("--template"); - else if (arg.startsWith("--template=")) template = arg.slice("--template=".length).trim(); - else if (arg === "--kind") kind = parseRuntimeKind(readValue("--kind")); - else if (arg.startsWith("--kind=")) kind = parseRuntimeKind(arg.slice("--kind=".length)); - else if (arg === "--target") targetId = readValue("--target"); - else if (arg.startsWith("--target=")) targetId = arg.slice("--target=".length).trim(); - else if (arg === "--since") since = readValue("--since"); - else if (arg.startsWith("--since=")) since = arg.slice("--since=".length).trim(); - else if (arg === "--tail") tail = parseRuntimeTail(readValue("--tail")); - else if (arg.startsWith("--tail=")) tail = parseRuntimeTail(arg.slice("--tail=".length)); - else throw new Error(`unsupported runtime option: ${arg}`); - } - if ((actionRaw === "get" || actionRaw === "apply" || actionRaw === "delete") && !account) { - throw new Error(`runtime ${actionRaw} requires --account `); - } - if (account !== null && (account.length > 256 || /[\r\n]/u.test(account))) throw new Error("--account has an unsupported format"); - if (group !== null && (group.length > 256 || /[\r\n]/u.test(group))) throw new Error("--group has an unsupported format"); - if (platform !== null && (!/^[A-Za-z0-9._-]+$/u.test(platform) || platform.length > 64)) throw new Error("--platform has an unsupported format"); - if (pageToken !== null && (!/^\d+$/u.test(pageToken) || pageToken.length > 20)) throw new Error("--page-token must be a numeric group cursor"); - if (group !== null && allGroups) throw new Error("use only one of --group or --all-groups"); - if ((group !== null || allGroups || platform !== null || pageToken !== null) && actionRaw !== "errors") throw new Error(`runtime ${actionRaw} does not accept group scope options`); - if (pageToken !== null && !allGroups) throw new Error("--page-token requires --all-groups"); - if (account !== null && allGroups) throw new Error("--account cannot be combined with --all-groups; use --group first"); - if (actionRaw === "apply" && !template) throw new Error("runtime apply requires --template "); - if (actionRaw !== "apply" && template !== null) throw new Error(`runtime ${actionRaw} does not accept --template`); - if (actionRaw !== "errors" && (since !== "24h" || tail !== 50_000)) throw new Error(`runtime ${actionRaw} does not accept --since or --tail`); - if (!/^\d+[mhd]$/u.test(since)) throw new Error("--since must use m, h, or d"); - if ((actionRaw === "list" || actionRaw === "get" || actionRaw === "errors") && confirm) throw new Error(`runtime ${actionRaw} does not accept --confirm`); - if ([full, raw, json].filter(Boolean).length > 1) throw new Error("use only one of --json, --full, or --raw"); - return { action: actionRaw, account, group, allGroups, platform, pageToken, template, kind, confirm, full, raw, json, since, tail, targetId }; -} - -function parseRuntimeKind(value: string): "temp-unschedulable" { - if (value !== "temp-unschedulable") throw new Error("--kind must be temp-unschedulable"); - return value; -} - -function parseRuntimeTail(value: string): number { - const parsed = Number(value); - if (!Number.isInteger(parsed) || parsed < 100 || parsed > 200_000) throw new Error("--tail must be an integer from 100 to 200000"); - return parsed; -} - function pyJson(value: unknown): string { return `json.loads(${JSON.stringify(JSON.stringify(value))})`; } @@ -571,10 +729,12 @@ set -u python3 - <<'PY' import json import re +import socket +import struct import subprocess import sys from datetime import datetime, timedelta, timezone -from urllib.parse import quote +from urllib.parse import quote, urlsplit RUNTIME_MODE = ${pyJson(target.runtimeMode)} NAMESPACE = ${pyJson(target.namespace)} @@ -790,6 +950,369 @@ def account_summary(detail, include_rules=False): result["tempUnschedulable"]["rules"] = policy["rules"] return result +def list_proxies(token): + return items_of(data_of(curl_api("GET", "/api/v1/admin/proxies?page=1&page_size=500", bearer=token), "list proxies")) + +def proxy_summary(token, proxy_id): + if proxy_id is None: + return {"configured": False, "route": "direct", "credentialsPrinted": False} + matches = [item for item in list_proxies(token) if isinstance(item, dict) and item.get("id") == proxy_id] + if len(matches) != 1: + return {"configured": True, "id": proxy_id, "status": "record-not-found", "credentialsPrinted": False} + item = matches[0] + return { + "configured": True, + "route": "account-proxy", + "id": item.get("id"), + "name": item.get("name"), + "protocol": item.get("protocol"), + "host": item.get("host"), + "port": item.get("port"), + "status": item.get("status"), + "fallbackMode": item.get("fallback_mode"), + "lastCheckAt": item.get("last_check_at"), + "lastError": text(item.get("last_error"), 240) if item.get("last_error") else None, + "credentialsPrinted": False, + } + +def proxy_environment_summary(): + rows = [] + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + value = app_text(["sh", "-c", f"printenv {name} 2>/dev/null || true"]) + parsed = urlsplit(value) if value else None + rows.append({ + "name": name, + "present": bool(value), + "scheme": parsed.scheme if parsed else None, + "host": parsed.hostname if parsed else None, + "port": parsed.port if parsed else None, + "credentialsRedacted": bool(parsed and (parsed.username or parsed.password)), + }) + no_proxy = app_text(["sh", "-c", "printenv NO_PROXY 2>/dev/null || printenv no_proxy 2>/dev/null || true"]) + return rows, { + "present": bool(no_proxy), + "entryCount": len([item for item in no_proxy.split(",") if item.strip()]), + "hyueapiPresent": any(item.strip() in ("hyueapi.com", ".hyueapi.com") for item in no_proxy.split(",")), + "valuesPrinted": False, + } + +def app_runtime_summary(): + if RUNTIME_MODE == "host-docker": + proc = docker(["inspect", APP_POD]) + if proc.returncode != 0: + return {"runtimeMode": RUNTIME_MODE, "name": APP_POD, "status": "unavailable", "reason": text(proc.stderr, 240)} + values = json.loads(proc.stdout.decode("utf-8")) + item = values[0] if values else {} + state = item.get("State") or {} + host_config = item.get("HostConfig") or {} + networks = ((item.get("NetworkSettings") or {}).get("Networks") or {}) + return { + "runtimeMode": RUNTIME_MODE, + "name": APP_POD, + "state": state.get("Status"), + "health": ((state.get("Health") or {}).get("Status")), + "restartCount": item.get("RestartCount"), + "oomKilled": state.get("OOMKilled"), + "startedAt": state.get("StartedAt"), + "networkMode": host_config.get("NetworkMode"), + "networks": [ + {"name": name, "ipAddress": value.get("IPAddress"), "gateway": value.get("Gateway")} + for name, value in sorted(networks.items()) if isinstance(value, dict) + ], + } + pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], "sub2api pod") + spec = pod.get("spec") or {} + status = pod.get("status") or {} + restarts = sum(item.get("restartCount") or 0 for item in status.get("containerStatuses") or [] if isinstance(item, dict)) + return { + "runtimeMode": RUNTIME_MODE, + "name": APP_POD, + "state": status.get("phase"), + "restartCount": restarts, + "oomKilled": None, + "startedAt": status.get("startTime"), + "networkMode": "host" if spec.get("hostNetwork") is True else "pod", + "nodeName": spec.get("nodeName"), + "podIp": status.get("podIP"), + "dnsPolicy": spec.get("dnsPolicy"), + } + +def dns_summary(): + content = app_text(["cat", "/etc/resolv.conf"]) + nameservers = [] + search = [] + options = [] + for line in content.splitlines(): + fields = line.strip().split() + if not fields or fields[0].startswith("#"): + continue + if fields[0] == "nameserver": nameservers.extend(fields[1:]) + elif fields[0] == "search": search.extend(fields[1:]) + elif fields[0] == "options": options.extend(fields[1:]) + return {"nameservers": nameservers, "search": search, "options": options} + +def default_route_summary(): + content = app_text(["cat", "/proc/net/route"]) + for line in content.splitlines()[1:]: + fields = line.split() + if len(fields) < 4 or fields[1] != "00000000": + continue + try: + gateway = socket.inet_ntoa(struct.pack("= 5 and peer_fields[0] == "ESTAB": + peers.append(peer_fields[4]) + udp_proc = run(["ss", "-unpH"]) + if udp_proc.returncode == 0: + for udp_line in udp_proc.stdout.decode("utf-8", errors="replace").splitlines(): + if f"pid={pid}," not in udp_line: + continue + udp_fields = udp_line.split() + if len(udp_fields) >= 5 and udp_fields[4] not in ("*:*", "0.0.0.0:*"): + udp_peers.append(udp_fields[4]) + cgroup_proc = run(["cat", f"/proc/{pid}/cgroup"]) + cgroup_text = cgroup_proc.stdout.decode("utf-8", errors="replace") if cgroup_proc.returncode == 0 else "" + unit_match = re.search(r'/([^/]+\\.service)(?:/|$)', cgroup_text) + systemd_unit = unit_match.group(1) if unit_match else None + if systemd_unit: + show_proc = run(["systemctl", "show", systemd_unit, "--property=ActiveState", "--property=SubState", "--property=NRestarts"]) + if show_proc.returncode == 0: + service_values = dict(line.split("=", 1) for line in show_proc.stdout.decode("utf-8", errors="replace").splitlines() if "=" in line) + active_state = "/".join(value for value in (service_values.get("ActiveState"), service_values.get("SubState")) if value) + service_restarts = service_values.get("NRestarts") + rows.append({ + "localEndpoint": local_endpoint, + "processName": process_name, + "pid": pid, + "executable": executable, + "establishedPeers": sorted(set(peers))[:12], + "udpPeers": sorted(set(udp_peers))[:12], + "systemdUnit": systemd_unit, + "activeState": active_state, + "serviceRestarts": service_restarts, + }) + return {"status": "available", "listeners": rows, "credentialsPrinted": False} + +def proxy_service_evidence(owner): + listeners = owner.get("listeners") if isinstance(owner, dict) else [] + units = sorted(set(item.get("systemdUnit") for item in listeners if isinstance(item, dict) and item.get("systemdUnit"))) + if not units: + return {"status": "not-applicable", "buckets": [], "samples": []} + buckets = {} + samples = [] + for unit in units: + proc = run(["journalctl", "--unit", unit, "--since", since_start_time(PAYLOAD.get("since")), "--no-pager", "--output", "json", "--lines", "2000"]) + if proc.returncode != 0: + continue + for line in proc.stdout.decode("utf-8", errors="replace").splitlines(): + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + message = str(item.get("MESSAGE") or "") + lowered = message.lower() + priority = int(item.get("PRIORITY")) if str(item.get("PRIORITY") or "").isdigit() else 7 + cause = None + if "reconnect" in lowered or "retry" in lowered: cause = "reconnect" + elif "disconnect" in lowered or "closed" in lowered: cause = "disconnected" + elif "timeout" in lowered or "deadline" in lowered: cause = "timeout" + elif "error" in lowered or "failed" in lowered or priority <= 3: cause = "service-error" + if cause is None: + continue + buckets[cause] = buckets.get(cause, 0) + 1 + if len(samples) < (30 if PAYLOAD.get("full") is True else 12): + samples.append({"at": item.get("__REALTIME_TIMESTAMP"), "cause": cause, "message": text(message, 400)}) + return { + "status": "available", + "units": units, + "buckets": [{"cause": key, "count": value} for key, value in sorted(buckets.items(), key=lambda pair: (-pair[1], pair[0]))], + "samples": samples, + } + +def classify_network_message(message, status_code): + lowered = message.lower() + checks = ( + ("dns-failure", ("no such host", "name resolution", "temporary failure in name resolution")), + ("proxy-connect-failure", ("proxyconnect", "connect tunnel", "http connect")), + ("connection-reset", ("connection reset", "reset by peer")), + ("unexpected-eof", ("unexpected eof", " eof")), + ("broken-pipe", ("broken pipe",)), + ("connection-refused", ("connection refused",)), + ("network-unreachable", ("network is unreachable", "no route to host")), + ("transport-timeout", ("i/o timeout", "tls handshake timeout", "context deadline exceeded")), + ("context-canceled", ("context canceled", "context cancelled")), + ("stream-incomplete", ("missing terminal event", "stream usage incomplete", "response failed")), + ) + for cause, markers in checks: + if any(marker in lowered for marker in markers): + return cause + return "upstream-503" if status_code == 503 or " 503" in lowered else None + +def recent_network_evidence(account_id): + lines = runtime_log_lines() + buckets = {} + samples = [] + matched = 0 + sample_limit = 30 if PAYLOAD.get("full") is True else 12 + for raw in lines: + candidate = raw[raw.find("{"):] if "{" in raw else raw + try: + item = json.loads(candidate) + except json.JSONDecodeError: + item = {} + extra = item.get("extra") if isinstance(item.get("extra"), dict) else {} + observed_account = item.get("account_id") if item.get("account_id") is not None else extra.get("account_id") + if str(observed_account) != str(account_id): + continue + status_code = item.get("status_code") if isinstance(item.get("status_code"), int) else extra.get("status_code") + if not isinstance(status_code, int): + upstream_status = extra.get("upstream_status") + status_code = upstream_status if isinstance(upstream_status, int) else None + message_parts = [item.get("message"), item.get("error"), extra.get("error"), extra.get("message")] + message = " | ".join(str(value) for value in message_parts if value not in (None, "")) + cause = classify_network_message(message, status_code) + if cause is None: + continue + matched += 1 + buckets[cause] = buckets.get(cause, 0) + 1 + if len(samples) < sample_limit: + samples.append({ + "at": item.get("time") or item.get("created_at") or extra.get("_at"), + "requestId": item.get("request_id") or extra.get("request_id"), + "cause": cause, + "statusCode": status_code, + "message": text(message, 360), + }) + return { + "source": "sub2api-target-runtime-logs", + "window": PAYLOAD.get("since"), + "scannedLines": len(lines), + "matchedEvents": matched, + "buckets": [{"cause": key, "count": value} for key, value in sorted(buckets.items(), key=lambda pair: (-pair[1], pair[0]))], + "samples": samples, + } + +def infrastructure_snapshot(token, detail): + proxy = proxy_summary(token, detail.get("proxy_id")) + proxy_env, no_proxy = proxy_environment_summary() + owner = proxy_endpoint_owner(proxy) + return { + "source": "sub2api-admin-api-and-target-runtime", + "accountProxy": proxy, + "appRuntime": app_runtime_summary(), + "proxyEnvironment": proxy_env, + "noProxy": no_proxy, + "dns": dns_summary(), + "defaultRoute": default_route_summary(), + "interfaces": interface_summaries(), + "proxySockets": proxy_socket_summary(proxy), + "proxyEndpointOwner": owner, + "proxyServiceEvidence": proxy_service_evidence(owner), + "recentNetworkEvidence": recent_network_evidence(detail.get("id")), + "attribution": "The account proxy record controls this account's transport. Container proxy environment is reported separately. Interface counters and sockets are query-time snapshots; no upstream account test or application request is sent.", + "valuesPrinted": False, + } + def policy_summary(policy, include_rules=False): keys = ("enabled", "ruleCount", "statusCodes", "rules") if include_rules else ("enabled", "ruleCount", "statusCodes") return {key: policy[key] for key in keys} @@ -1305,9 +1828,27 @@ def native_customer_error_profile(token, group_id, platform): counts[label] = counts.get(label, 0) + 1 return [{"value": key, "count": count} for key, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20 if PAYLOAD.get("full") is True else 8]] samples = [] + account_metrics = {} seen = set() for row in rows: - if not isinstance(row, dict) or not row.get("request_id"): + if not isinstance(row, dict): + continue + account_id = row.get("account_id") + account_key = str(account_id) if account_id is not None else "unknown" + if account_key not in account_metrics: + account_metrics[account_key] = { + "accountId": account_id, "customerErrorEvents": 0, "customerErrorRequestIds": set(), "statusCodes": {}, + } + metric = account_metrics[account_key] + metric["customerErrorEvents"] += 1 + request_id = row.get("request_id") + if isinstance(request_id, str) and request_id: + metric["customerErrorRequestIds"].add(request_id) + status_code = row.get("status_code") + if isinstance(status_code, int): + status_key = str(status_code) + metric["statusCodes"][status_key] = metric["statusCodes"].get(status_key, 0) + 1 + if not request_id: continue key = (row.get("requested_model") or row.get("model"), row.get("status_code"), row.get("account_id"), row.get("stream"), row.get("request_path") or row.get("inbound_endpoint")) if key in seen: @@ -1330,9 +1871,190 @@ def native_customer_error_profile(token, group_id, platform): "phaseBuckets": buckets(lambda row: row.get("phase")), "endpointBuckets": buckets(lambda row: row.get("inbound_endpoint") or row.get("request_path")), "pathBuckets": buckets(lambda row: row.get("request_path") or row.get("inbound_endpoint")), + "accountMetrics": [ + { + "accountId": item["accountId"], "customerErrorEvents": item["customerErrorEvents"], + "customerErrorRequests": len(item["customerErrorRequestIds"]), + "_requestIds": sorted(item["customerErrorRequestIds"]), + "statusBuckets": [ + {"statusCode": int(code), "count": count} + for code, count in sorted(item["statusCodes"].items(), key=lambda pair: int(pair[0])) + ], + } + for item in sorted(account_metrics.values(), key=lambda item: int(item.get("accountId") or 0)) + ], "samples": samples, } +def parse_native_time(value): + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc) + except ValueError: + return None + +def metric_percentile(values, fraction): + ordered = sorted(value for value in values if isinstance(value, (int, float)) and value >= 0) + if not ordered: + return None + position = (len(ordered) - 1) * fraction + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + return round(ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)) + +def account_usage_metrics(token, accounts, group_id): + start_at = parse_native_time(since_start_time(PAYLOAD.get("since"))) + end_at = datetime.now(timezone.utc) + result = {} + for account in accounts: + account_id = account.get("id") + key = str(account_id) + rows = [] + page = 1 + status = "available" + reason = None + try: + while True: + path = ( + f"/api/v1/admin/usage?page={page}&page_size=100&group_id={group_id}&account_id={account_id}" + f"&start_date={start_at.date().isoformat()}&end_date={end_at.date().isoformat()}&timezone=UTC" + "&sort_by=created_at&sort_order=desc" + ) + data = data_of(curl_api("GET", path, bearer=token), "list account usage") + page_rows = items_of(data) + reached_start = False + for row in page_rows: + created_at = parse_native_time(row.get("created_at") if isinstance(row, dict) else None) + if created_at is None or created_at < start_at or created_at > end_at: + if created_at is not None and created_at < start_at: + reached_start = True + continue + rows.append(row) + if not page_rows or len(page_rows) < 100 or reached_start: + break + page += 1 + except Exception as exc: + status = "unavailable" + reason = compact_error(str(exc)) + stream_rows = [row for row in rows if isinstance(row, dict) and row.get("stream") is True] + ttft_values = [row.get("first_token_ms") for row in stream_rows if isinstance(row.get("first_token_ms"), int)] + duration_values = [row.get("duration_ms") for row in rows if isinstance(row.get("duration_ms"), int)] + models = {} + for row in rows: + model = str(row.get("model") or "unknown") + models[model] = models.get(model, 0) + 1 + result[key] = { + "status": status, "reason": reason, "source": "sub2api-native-admin-usage", + "pagesRead": page, "successRequests": len(rows), "streamSuccessRequests": len(stream_rows), + "firstTokenSamples": len(ttft_values), + "firstTokenCoverage": round(len(ttft_values) / len(stream_rows), 6) if stream_rows else None, + "ttftP50Ms": metric_percentile(ttft_values, 0.50), "ttftP95Ms": metric_percentile(ttft_values, 0.95), + "ttftP99Ms": metric_percentile(ttft_values, 0.99), "ttftMaxMs": max(ttft_values) if ttft_values else None, + "durationP95Ms": metric_percentile(duration_values, 0.95), + "modelBuckets": [{"model": model, "count": count} for model, count in sorted(models.items(), key=lambda item: (-item[1], item[0]))[:8]], + } + return result + +def reliability_score_points(failure_rate): + if not isinstance(failure_rate, (int, float)): + return None + return round(60 * (1 - min(max(failure_rate, 0), 0.20) / 0.20), 2) + +def latency_score_points(ttft_p95_ms): + if not isinstance(ttft_p95_ms, (int, float)): + return None + return round(25 * (1 - min(max(ttft_p95_ms - 10000, 0), 170000) / 170000), 2) + +def account_quality_scores(token, accounts, group_id, native_ops, effectiveness): + usage_by_account = account_usage_metrics(token, accounts, group_id) + customer_profile = native_ops.get("customerErrors") if isinstance(native_ops.get("customerErrors"), dict) else {} + customer_by_account = {str(item.get("accountId")): item for item in customer_profile.get("accountMetrics") or [] if isinstance(item, dict)} + policy_by_account = {str(item.get("accountId")): item for item in effectiveness.get("accountPolicyMetrics") or [] if isinstance(item, dict)} + availability = ((native_ops.get("accountAvailability") or {}).get("summary") or {}) + unavailable_ids = set(str(item.get("accountId")) for item in availability.get("unavailableAccounts") or [] if isinstance(item, dict)) + rows = [] + for account in accounts: + account_id = account.get("id") + key = str(account_id) + usage = usage_by_account.get(key) or {} + customer = customer_by_account.get(key) or {} + policy = policy_by_account.get(key) or {} + policy_failure_ids = set(policy.pop("_failureRequestIds", []) or []) + customer_failure_ids = set(customer.pop("_requestIds", []) or []) + failure_requests = len(policy_failure_ids | customer_failure_ids) + success_requests = usage.get("successRequests") if isinstance(usage.get("successRequests"), int) else 0 + observed_attempts = success_requests + failure_requests + failure_rate = round(failure_requests / observed_attempts, 6) if observed_attempts > 0 else None + reliability_points = reliability_score_points(failure_rate) + ttft_samples = usage.get("firstTokenSamples") if isinstance(usage.get("firstTokenSamples"), int) else 0 + latency_points = latency_score_points(usage.get("ttftP95Ms")) if ttft_samples >= 5 else None + current_available = key not in unavailable_ids and account.get("status") == "active" and account.get("schedulable") is not False + availability_points = 15 if current_available else 8 if account.get("status") == "active" else 0 + earned = (reliability_points or 0) + (latency_points or 0) + availability_points + available_weight = (60 if reliability_points is not None else 0) + (25 if latency_points is not None else 0) + 15 + score = round(earned / available_weight * 100, 1) if observed_attempts > 0 and available_weight > 0 else None + score_comparable = observed_attempts >= 10 and ttft_samples >= 5 + grade = "insufficient" + if isinstance(score, (int, float)) and (score_comparable or (score < 60 and observed_attempts >= 10)): + grade = "A" if score >= 90 else "B" if score >= 80 else "C" if score >= 70 else "D" if score >= 60 else "E" + assessment = "preferred" if grade == "A" else "healthy" if grade == "B" else "watch" if grade == "C" else "degraded" if grade == "D" else "poor" if grade == "E" else "insufficient-evidence" + confidence = "high" if observed_attempts >= 50 and ttft_samples >= 20 else "medium" if observed_attempts >= 10 and ttft_samples >= 5 else "low" + reasons = [] + if isinstance(failure_rate, (int, float)) and failure_rate >= 0.10: + reasons.append("failure-rate>=10%") + elif isinstance(failure_rate, (int, float)) and failure_rate >= 0.03: + reasons.append("failure-rate>=3%") + if isinstance(usage.get("ttftP95Ms"), int) and usage.get("ttftP95Ms") >= 120000: + reasons.append("ttft-p95>=120s") + if policy.get("failoverRequests", 0) > 0: + reasons.append("upstream-failover-triggered") + if policy.get("sameAccountRetryEvents", 0) > 0: + reasons.append("same-account-retry-observed") + if not current_available: + reasons.append("currently-unavailable") + if ttft_samples < 5: + reasons.append("ttft-evidence-insufficient") + if observed_attempts < 10: + reasons.append("request-evidence-insufficient") + rows.append({ + "accountId": account_id, "accountName": account.get("name"), "status": account.get("status"), + "schedulable": account.get("schedulable"), "currentlyAvailable": current_available, + "score": score, "grade": grade, "assessment": assessment, "confidence": confidence, + "scoreComparable": score_comparable, + "observedAttempts": observed_attempts, "successRequests": success_requests, + "failureRequests": failure_requests, "failureRate": failure_rate, + "streamSuccessRequests": usage.get("streamSuccessRequests"), "firstTokenSamples": ttft_samples, + "firstTokenCoverage": usage.get("firstTokenCoverage"), "ttftP50Ms": usage.get("ttftP50Ms"), + "ttftP95Ms": usage.get("ttftP95Ms"), "ttftP99Ms": usage.get("ttftP99Ms"), "ttftMaxMs": usage.get("ttftMaxMs"), + "durationP95Ms": usage.get("durationP95Ms"), "modelBuckets": usage.get("modelBuckets"), + "customerErrorRequests": customer.get("customerErrorRequests", 0), + "failoverRequests": policy.get("failoverRequests", 0), "failoverRecovered": policy.get("failoverRecovered", 0), + "failoverFailed": policy.get("failoverFailed", 0), "failoverOutcomeMissing": policy.get("failoverOutcomeMissing", 0), + "sameAccountRetryEvents": policy.get("sameAccountRetryEvents", 0), + "tempUnschedulableEvents": policy.get("tempUnschedulableEvents", 0), + "forwardFailedRequests": policy.get("forwardFailedRequests", 0), + "upstreamStatusBuckets": policy.get("upstreamStatusBuckets", []), + "scoreComponents": {"reliability": reliability_points, "latency": latency_points, "availability": availability_points, "availableWeight": available_weight}, + "reasons": reasons, "usageStatus": usage.get("status"), "usageReason": usage.get("reason"), + }) + rows.sort(key=lambda item: (item.get("score") is None, -(item.get("score") or -1), int(item.get("accountId") or 0))) + return { + "source": "sub2api-native-admin-usage-request-errors-and-system-logs", + "scope": "group-account-window", "window": PAYLOAD.get("since"), + "scoreRule": { + "reliabilityWeight": 60, "latencyWeight": 25, "availabilityWeight": 15, + "failureDefinition": "unique request ids with account failover, forward_failed, or customer-visible error", + "reliability": "linear from 60 points at 0% observed failures to 0 points at 20%", + "latency": "requires at least 5 first-token samples; linear from 25 points at <=10s TTFT P95 to 0 points at >=180s", + "availability": "15 points when active, schedulable, and currently available; 8 when active but unavailable; otherwise 0", + "missingMetric": "missing latency is excluded from the denominator and lowers confidence; it is not treated as zero latency", + }, + "accounts": rows, + "attribution": "Success denominators and first-token latency come from native admin usage records. Failures are deduplicated by request id across native request-errors and indexed failover/forward events. Scores are analytical and never mutate scheduling.", + } + def all_group_error_overview(token, groups): page_size = 20 cursor = int(PAYLOAD.get("pageToken") or 0) @@ -1440,15 +2162,33 @@ def native_system_log_lines(token, platform=None): platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" markers = ( "account_temp_unschedulable", "openai.upstream_failover_switching", - "openai.account_select_failed", "openai.forward_failed", "http request completed", + "openai.account_select_failed", "openai.forward_failed", + "openai.pool_mode_same_account_retry", "gateway.failover_same_account_retry", + "http request completed", ) rows_by_id = {} queries = [] for marker in markers: - path = f"/api/v1/admin/ops/system-logs?page=1&page_size=200&start_time={start_time}&q={quote(marker, safe='')}{platform_query}" - response = curl_api("GET", path, bearer=token) + page = 1 + returned = 0 + total = None try: - data = data_of(response, "list ops system logs") + while True: + path = f"/api/v1/admin/ops/system-logs?page={page}&page_size=200&start_time={start_time}&q={quote(marker, safe='')}{platform_query}" + data = data_of(curl_api("GET", path, bearer=token), "list ops system logs") + rows = items_of(data) + returned += len(rows) + if total is None and isinstance(data, dict) and isinstance(data.get("total"), int): + total = data.get("total") + for row in rows: + if not isinstance(row, dict): + continue + row_id = row.get("id") + key = str(row_id) if row_id is not None else json.dumps(row, sort_keys=True, default=str) + rows_by_id[key] = row + if not rows or len(rows) < 200 or (isinstance(total, int) and returned >= total): + break + page += 1 except Exception as exc: return None, { "status": "unavailable", @@ -1456,15 +2196,7 @@ def native_system_log_lines(token, platform=None): "reason": compact_error(str(exc)), "fallback": "runtime-container-logs", } - rows = items_of(data) - total = data.get("total", len(rows)) if isinstance(data, dict) else len(rows) - queries.append({"marker": marker, "returned": len(rows), "total": total, "truncated": total > len(rows)}) - for row in rows: - if not isinstance(row, dict): - continue - row_id = row.get("id") - key = str(row_id) if row_id is not None else json.dumps(row, sort_keys=True, default=str) - rows_by_id[key] = row + queries.append({"marker": marker, "returned": returned, "total": total if total is not None else returned, "pagesRead": page, "truncated": False}) lines = [] for row in sorted(rows_by_id.values(), key=lambda item: str(item.get("created_at") or "")): extra = dict(row.get("extra") or {}) if isinstance(row.get("extra"), dict) else {} @@ -1487,12 +2219,23 @@ def native_system_log_lines(token, platform=None): def policy_effectiveness(lines, accounts, source, group_id=None): account_names = {str(item.get("id")): item.get("name") for item in accounts} account_ids = set(account_names.keys()) + account_policy = { + key: { + "accountId": item.get("id"), "accountName": item.get("name"), + "tempUnschedulableEvents": 0, "failoverEvents": 0, "failoverRequestIds": set(), + "failoverRecoveredRequestIds": set(), "failoverFailedRequestIds": set(), "failoverOutcomeMissingRequestIds": set(), + "forwardFailedEvents": 0, "forwardFailedRequestIds": set(), "sameAccountRetryEvents": 0, + "upstreamStatusCodes": {}, + } + for key, item in ((str(item.get("id")), item) for item in accounts) + } requests = {} counts = { "tempUnschedulableEvents": 0, "failoverEvents": 0, "selectFailedEvents": 0, "forwardFailedEvents": 0, + "sameAccountRetryEvents": 0, "gatewayCompletedEvents": 0, "gatewayFinalErrorEvents": 0, } @@ -1501,6 +2244,7 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "failover": 0, "selectFailed": 0, "forwardFailed": 0, + "sameAccountRetry": 0, "gatewayCompleted": 0, } temp_accounts = {} @@ -1520,6 +2264,7 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "failoverCount": 0, "selectFailedCount": 0, "forwardFailedCount": 0, + "sameAccountRetryCount": 0, "finalStatus": None, } return requests[request_id] @@ -1527,7 +2272,9 @@ def policy_effectiveness(lines, accounts, source, group_id=None): for line in lines: if not any(marker in line for marker in ( "account_temp_unschedulable", "openai.upstream_failover_switching", - "openai.account_select_failed", "openai.forward_failed", "http request completed", + "openai.account_select_failed", "openai.forward_failed", + "openai.pool_mode_same_account_retry", "gateway.failover_same_account_retry", + "http request completed", )): continue json_start = line.find("{") @@ -1557,6 +2304,8 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "accountId": account_id, "accountName": account_names.get(str(account_id)), } + raw_account_stat = account_policy.get(str(account_id)) + account_stat = raw_account_stat if group_id is None or event_group_id is not None else None if "account_temp_unschedulable" in line: event_line_count += 1 counts["tempUnschedulableEvents"] += 1 @@ -1572,6 +2321,8 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "count": 0, } temp_accounts[key]["count"] += 1 + if raw_account_stat is not None: + raw_account_stat["tempUnschedulableEvents"] += 1 if len(temp_samples) < 20: temp_samples.append({ **base, @@ -1588,6 +2339,14 @@ def policy_effectiveness(lines, accounts, source, group_id=None): request["failoverCount"] += 1 else: missing_request_ids["failover"] += 1 + if account_stat is not None: + account_stat["failoverEvents"] += 1 + if isinstance(request_id, str) and request_id: + account_stat["failoverRequestIds"].add(request_id) + status_code = item.get("upstream_status") + if isinstance(status_code, int): + key = str(status_code) + account_stat["upstreamStatusCodes"][key] = account_stat["upstreamStatusCodes"].get(key, 0) + 1 if len(failover_samples) < 20: failover_samples.append({ **base, @@ -1614,6 +2373,19 @@ def policy_effectiveness(lines, accounts, source, group_id=None): request["forwardFailedCount"] += 1 else: missing_request_ids["forwardFailed"] += 1 + if account_stat is not None: + account_stat["forwardFailedEvents"] += 1 + if isinstance(request_id, str) and request_id: + account_stat["forwardFailedRequestIds"].add(request_id) + elif "openai.pool_mode_same_account_retry" in line or "gateway.failover_same_account_retry" in line: + event_line_count += 1 + counts["sameAccountRetryEvents"] += 1 + if request is not None: + request["sameAccountRetryCount"] += 1 + else: + missing_request_ids["sameAccountRetry"] += 1 + if raw_account_stat is not None: + raw_account_stat["sameAccountRetryEvents"] += 1 elif "http request completed" in line and gateway_request_path(item.get("path")): event_line_count += 1 counts["gatewayCompletedEvents"] += 1 @@ -1645,6 +2417,37 @@ def policy_effectiveness(lines, accounts, source, group_id=None): item for item in requests.values() if item["forwardFailedCount"] > 0 and not isinstance(item.get("finalStatus"), int) ] + for request in correlated: + request_id = request.get("requestId") + if not isinstance(request_id, str): + continue + for current in account_policy.values(): + if request_id not in current["failoverRequestIds"]: + continue + final_status = request.get("finalStatus") + if isinstance(final_status, int) and final_status < 400: + current["failoverRecoveredRequestIds"].add(request_id) + elif isinstance(final_status, int): + current["failoverFailedRequestIds"].add(request_id) + else: + current["failoverOutcomeMissingRequestIds"].add(request_id) + account_policy_rows = [] + for current in account_policy.values(): + account_policy_rows.append({ + "accountId": current["accountId"], "accountName": current["accountName"], + "tempUnschedulableEvents": current["tempUnschedulableEvents"], + "failoverEvents": current["failoverEvents"], "failoverRequests": len(current["failoverRequestIds"]), + "failoverRecovered": len(current["failoverRecoveredRequestIds"]), + "failoverFailed": len(current["failoverFailedRequestIds"]), + "failoverOutcomeMissing": len(current["failoverOutcomeMissingRequestIds"]), + "forwardFailedEvents": current["forwardFailedEvents"], "forwardFailedRequests": len(current["forwardFailedRequestIds"]), + "sameAccountRetryEvents": current["sameAccountRetryEvents"], + "_failureRequestIds": sorted(current["failoverRequestIds"] | current["forwardFailedRequestIds"]), + "upstreamStatusBuckets": [ + {"statusCode": int(code), "count": count} + for code, count in sorted(current["upstreamStatusCodes"].items(), key=lambda pair: int(pair[0])) + ], + }) sample_limit = 20 if PAYLOAD.get("full") is True else (2 if PAYLOAD.get("account") is not None else 5) completed = counts["gatewayCompletedEvents"] final_errors = counts["gatewayFinalErrorEvents"] @@ -1655,6 +2458,8 @@ def policy_effectiveness(lines, accounts, source, group_id=None): "eventLineCount": event_line_count, **counts, "tempUnschedulableAccounts": sorted(temp_accounts.values(), key=lambda item: (-item["count"], str(item.get("accountName") or ""))), + "accountPolicyMetrics": sorted(account_policy_rows, key=lambda item: int(item.get("accountId") or 0)), + "accountPolicyAttribution": "Failover and forward-failure request ids enter group scoring only when system-log events carry group_id. Account temp-unschedulable and same-account retry counts are account-wide operational signals and are not added to the failure-rate numerator.", "failoverRequestCount": len(correlated), "failoverSucceededRequestCount": len(succeeded), "failoverFailedRequestCount": len(failed), @@ -1746,12 +2551,13 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): system_log_source["fallbackUsed"] = True system_log_source["fallbackReason"] = "native system-log index returned no policy events while the native overview reported upstream errors" system_log_source["fallbackEventCount"] = len(runtime_event_lines) - effectiveness = policy_effectiveness(lines, accounts, system_log_source.get("source"), group_id) + effectiveness = policy_effectiveness(lines, selected if selector is not None else accounts, system_log_source.get("source"), group_id) effectiveness["sourceStatus"] = system_log_source overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) native_ops["responseHeaderTimeout"] = runtime_response_header_timeout() native_ops["requestTiming"] = native_request_timing(token, group_id, platform, overview_summary) native_ops["customerErrors"] = native_customer_error_profile(token, group_id, platform) + native_ops["accountQuality"] = account_quality_scores(token, selected, group_id, native_ops, effectiveness) request_total = overview_summary.get("request_count_total") error_total = overview_summary.get("error_count_total") if isinstance(request_total, int) and isinstance(error_total, int): @@ -1888,6 +2694,7 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): "responseHeaderTimeout": native_ops.get("responseHeaderTimeout"), "requestTiming": native_ops.get("requestTiming"), "customerErrors": native_ops.get("customerErrors"), + "accountQuality": native_ops.get("accountQuality"), "accountAvailability": { "status": (native_ops.get("accountAvailability") or {}).get("status"), "group": availability.get("group"), @@ -1981,6 +2788,37 @@ def observed_runtime_errors(token, accounts, group_id, platform=None): "message": "observedStatusCodes comes only from account_upstream_error; forward failures are supplemental message evidence", } +def runtime_mutation_plan(detail, action): + if detail.get("type") != "apikey": + raise RuntimeError(f"runtime temp-unschedulable policy requires an apikey account: {detail.get('name')} ({detail.get('id')})") + credentials = dict(detail.get("credentials") or {}) + before = normalize_policy(credentials) + if action == "apply": + template = PAYLOAD.get("selectedTemplate") + if not isinstance(template, dict): + raise RuntimeError("selected runtime template missing") + desired = dict(credentials) + desired.update(template.get("credentials") or {}) + after = normalize_policy(desired) + exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials + change = "noop" if before == after else ("update" if exists else "create") + else: + desired = dict(credentials) + desired.pop("temp_unschedulable_enabled", None) + desired.pop("temp_unschedulable_rules", None) + after = normalize_policy(desired) + change = "delete" if before != after or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop" + include_rules = PAYLOAD.get("full") is True + plan = { + "change": change, + "account": account_summary(detail, include_rules), + "before": policy_summary(before, include_rules), + "after": policy_summary(after, include_rules), + "template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None, + "credentialsPrinted": False, + } + return plan, desired + def runtime_result(): token = login() action = PAYLOAD["action"] @@ -1996,8 +2834,8 @@ def runtime_result(): "valuesPrinted": False, } group_selector = PAYLOAD.get("group") - group_platform = PAYLOAD.get("platform") if action == "errors" else "openai" - if action == "errors" and group_selector is None: + group_platform = PAYLOAD.get("platform") if action in ("errors", "infrastructure") else "openai" + if action in ("errors", "infrastructure") and group_selector is None: group_platform = "openai" groups = list_groups(token, group_platform) selector = str(group_selector if group_selector is not None else PAYLOAD["groupName"]) @@ -2021,43 +2859,49 @@ def runtime_result(): if action == "errors": errors = observed_runtime_errors(token, accounts, group["id"], effective_platform) return {**base, "ok": True, "operation": "errors", "mutation": False, "errors": errors} - detail = find_account(token, accounts, str(PAYLOAD["account"])) + selectors = PAYLOAD.get("accounts") or [PAYLOAD.get("account")] + details = [find_account(token, accounts, str(item)) for item in selectors if item is not None] + if len({item.get("id") for item in details}) != len(details): + raise RuntimeError("account selectors resolve to duplicate accounts") + detail = details[0] if action == "get": return {**base, "ok": True, "operation": "get", "mutation": False, "account": account_summary(detail, True)} - if detail.get("type") != "apikey": - raise RuntimeError("runtime temp-unschedulable policy requires an apikey account") - credentials = dict(detail.get("credentials") or {}) - before = normalize_policy(credentials) - if action == "apply": - template = PAYLOAD.get("selectedTemplate") - if not isinstance(template, dict): - raise RuntimeError("selected runtime template missing") - desired_fields = template.get("credentials") or {} - desired = dict(credentials) - desired.update(desired_fields) - after_plan = normalize_policy(desired) - exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials - change = "noop" if before == after_plan else ("update" if exists else "create") - else: - desired = dict(credentials) - desired.pop("temp_unschedulable_enabled", None) - desired.pop("temp_unschedulable_rules", None) - after_plan = normalize_policy(desired) - change = "delete" if before != after_plan or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop" - include_plan_rules = PAYLOAD.get("full") is True - plan = { - "change": change, - "account": account_summary(detail, include_plan_rules), - "before": policy_summary(before, include_plan_rules), - "after": policy_summary(after_plan, include_plan_rules), - "template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None, - "credentialsPrinted": False, - } - if PAYLOAD.get("confirm") is not True or change == "noop": - return {**base, "ok": True, "operation": action, "mode": "dry-run" if PAYLOAD.get("confirm") is not True else "confirmed-noop", "mutation": False, "plan": plan} - updated = data_of(curl_api("PUT", f"/api/v1/admin/accounts/{detail['id']}", bearer=token, payload={"credentials": desired}), "update account runtime config") - after_detail = updated if isinstance(updated, dict) else account_detail(token, detail["id"]) - return {**base, "ok": True, "operation": action, "mode": "confirmed", "mutation": True, "plan": plan, "account": account_summary(after_detail, include_plan_rules)} + if action == "infrastructure": + return { + **base, + "ok": True, + "operation": "infrastructure", + "mutation": False, + "account": account_summary(detail, False), + "infrastructure": infrastructure_snapshot(token, detail), + } + work = [(item, *runtime_mutation_plan(item, action)) for item in details] + if len(work) == 1: + item, plan, desired = work[0] + if PAYLOAD.get("confirm") is not True or plan["change"] == "noop": + return {**base, "ok": True, "operation": action, "mode": "dry-run" if PAYLOAD.get("confirm") is not True else "confirmed-noop", "mutation": False, "plan": plan} + updated = data_of(curl_api("PUT", f"/api/v1/admin/accounts/{item['id']}", bearer=token, payload={"credentials": desired}), "update account runtime config") + after_detail = updated if isinstance(updated, dict) else account_detail(token, item["id"]) + return {**base, "ok": True, "operation": action, "mode": "confirmed", "mutation": True, "plan": plan, "account": account_summary(after_detail, PAYLOAD.get("full") is True)} + changed = sum(1 for _, plan, _ in work if plan["change"] != "noop") + if PAYLOAD.get("confirm") is not True: + plans = [{**plan, "result": "planned" if plan["change"] != "noop" else "noop"} for _, plan, _ in work] + return {**base, "ok": True, "operation": action, "mode": "dry-run", "mutation": False, "plans": plans, "summary": {"selected": len(work), "changed": changed, "succeeded": 0, "failed": 0}} + succeeded = 0 + failed = 0 + plans = [] + for item, plan, desired in work: + if plan["change"] == "noop": + plans.append({**plan, "result": "noop"}) + continue + try: + data_of(curl_api("PUT", f"/api/v1/admin/accounts/{item['id']}", bearer=token, payload={"credentials": desired}), "update account runtime config") + succeeded += 1 + plans.append({**plan, "result": "succeeded"}) + except Exception as exc: + failed += 1 + plans.append({**plan, "result": "failed: " + str(exc)}) + return {**base, "ok": failed == 0, "operation": action, "mode": "confirmed" if failed == 0 else "confirmed-partial", "mutation": succeeded > 0, "plans": plans, "summary": {"selected": len(work), "changed": changed, "succeeded": succeeded, "failed": failed}} try: result = runtime_result()