From 41c1d59ca0d17c57c51693f2b478685c05e8ae81 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:40:23 +0200 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=20Sub2API=20?= =?UTF-8?q?=E6=95=85=E9=9A=9C=E7=AD=89=E7=BA=A7=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform-infra-sub2api-codex/faults.ts | 569 ++++++++++++++++++ .../platform-infra-sub2api-codex/options.ts | 2 + .../src/platform-infra-sub2api-codex/types.ts | 3 +- 3 files changed, 573 insertions(+), 1 deletion(-) create mode 100644 scripts/src/platform-infra-sub2api-codex/faults.ts diff --git a/scripts/src/platform-infra-sub2api-codex/faults.ts b/scripts/src/platform-infra-sub2api-codex/faults.ts new file mode 100644 index 00000000..8f89e410 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/faults.ts @@ -0,0 +1,569 @@ +import { createHash } from "node:crypto"; + +import type { UniDeskConfig } from "../config"; +import { CliInputError, type RenderedCliResult } from "../output"; + +import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote"; +import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target"; + +type FaultLevel = "P0" | "P1" | "P2"; + +interface FaultOptions { + level: FaultLevel | null; + group: string | null; + account: string | null; + model: string | null; + stream: "sync" | "stream" | null; + endpoint: string | null; + requestId: string | null; + pageToken: string | null; + targetId: string; + json: boolean; +} + +const PAGE_SIZE = 20; + +export async function codexPoolFaults(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { + if (args.includes("--help")) return renderFaultHelp(); + const options = parseFaultOptions(args); + const target = codexPoolRuntimeTarget(options.targetId); + const scope = faultScope(options); + const offset = decodePageToken(options.pageToken, scope); + const payload = { + filters: { + level: options.level, + group: options.group, + account: options.account, + model: options.model, + stream: options.stream, + endpoint: options.endpoint, + requestId: options.requestId, + }, + offset, + pageSize: PAGE_SIZE, + }; + const result = await runRemoteCodexPoolScript(config, "faults", faultScript(payload, target), target); + const parsed = parseJsonOutput(result.stdout); + const ok = result.exitCode === 0 && boolField(parsed, "ok", false); + const data = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : null; + const pagination = data && typeof data.pagination === "object" && data.pagination !== null + ? data.pagination as Record + : null; + if (pagination?.hasMore === true) pagination.nextPageToken = encodePageToken(offset + PAGE_SIZE, scope); + if (pagination) { + pagination.pageSize = PAGE_SIZE; + pagination.pageToken = options.pageToken; + } + const response = { + ok, + action: "platform-infra-sub2api-codex-pool-faults", + target: { + id: target.id, + route: target.route, + runtimeMode: target.runtimeMode, + endpoint: target.serviceDns, + }, + source: { + kind: "sub2api-native-admin-ops", + versionBoundary: "Sub2API native admin Ops facts with explicit, version-neutral UniDesk CLI projection", + mutation: false, + }, + filters: payload.filters, + faults: data, + remote: compactCapture(result, { full: result.exitCode !== 0 || data === null }), + valuesPrinted: false, + }; + if (options.json) return response; + return renderFaults(response); +} + +function renderFaultHelp(): RenderedCliResult { + return { + ok: true, + command: "platform-infra sub2api codex-pool faults --help", + renderedText: [ + "SUB2API CODEX-POOL FAULTS", + "Usage:", + " bun scripts/cli.ts platform-infra sub2api codex-pool faults [options]", + "Options:", + " --level P0|P1|P2", + " --group ", + " --account ", + " --model ", + " --stream sync|stream", + " --endpoint ", + " --request-id ", + " --page-token ", + " --target ", + " --json", + "Notes:", + ` Fixed page size: ${PAGE_SIZE}; --limit is intentionally unsupported.`, + " Default output is a bounded Kubernetes-style table; JSON requires --json.", + ].join("\n"), + contentType: "text/plain", + }; +} + +function parseFaultOptions(args: string[]): FaultOptions { + let level: FaultLevel | null = null; + let group: string | null = null; + let account: string | null = null; + let model: string | null = null; + let stream: "sync" | "stream" | null = null; + let endpoint: string | null = null; + let requestId: string | null = null; + let pageToken: string | null = null; + let targetId = defaultCodexPoolRuntimeTargetId(); + let json = false; + const readValue = (index: number, name: string): [string, number] => { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); + return [validateSelector(value, name), index + 1]; + }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]!; + if (arg === "--json") json = true; + else if (arg === "--level") { + const [value, next] = readValue(index, "--level"); + level = parseLevel(value); + index = next; + } else if (arg.startsWith("--level=")) level = parseLevel(arg.slice(8)); + else if (arg === "--group") [group, index] = readValue(index, "--group"); + else if (arg.startsWith("--group=")) group = validateSelector(arg.slice(8), "--group"); + else if (arg === "--account") [account, index] = readValue(index, "--account"); + else if (arg.startsWith("--account=")) account = validateSelector(arg.slice(10), "--account"); + else if (arg === "--model") [model, index] = readValue(index, "--model"); + else if (arg.startsWith("--model=")) model = validateSelector(arg.slice(8), "--model"); + else if (arg === "--stream") { + const [value, next] = readValue(index, "--stream"); + stream = parseStream(value); + index = next; + } else if (arg.startsWith("--stream=")) stream = parseStream(arg.slice(9)); + else if (arg === "--endpoint") [endpoint, index] = readValue(index, "--endpoint"); + else if (arg.startsWith("--endpoint=")) endpoint = validateSelector(arg.slice(11), "--endpoint"); + else if (arg === "--request-id") [requestId, index] = readValue(index, "--request-id"); + else if (arg.startsWith("--request-id=")) requestId = validateSelector(arg.slice(13), "--request-id"); + else if (arg === "--page-token") [pageToken, index] = readValue(index, "--page-token"); + else if (arg.startsWith("--page-token=")) pageToken = validateSelector(arg.slice(13), "--page-token"); + else if (arg === "--target") [targetId, index] = readValue(index, "--target"); + else if (arg.startsWith("--target=")) targetId = validateSelector(arg.slice(9), "--target"); + else if (arg === "--limit" || arg.startsWith("--limit=")) throw new Error("faults uses a fixed page size; use --page-token for progressive disclosure"); + else throw new Error(`unsupported faults option: ${arg}`); + } + if (level !== null && group === null) { + throw new CliInputError("--level requires --group; use the default command for the all-groups overview", { + code: "missing-required-option", + argument: "--group", + usage: [ + "bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01", + "bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01 --level P0 --group ", + ], + hint: "Run without --level for the all-groups overview, then drill down with --level and --group.", + }); + } + return { level, group, account, model, stream, endpoint, requestId, pageToken, targetId, json }; +} + +function parseLevel(value: string): FaultLevel { + const normalized = value.toUpperCase(); + if (normalized !== "P0" && normalized !== "P1" && normalized !== "P2") throw new Error("--level must be P0, P1, or P2"); + return normalized; +} + +function parseStream(value: string): "sync" | "stream" { + if (value !== "sync" && value !== "stream") throw new Error("--stream must be sync or stream"); + return value; +} + +function validateSelector(value: string, name: string): string { + const normalized = value.trim(); + if (!normalized || normalized.length > 512 || /[\r\n\0]/u.test(normalized)) throw new Error(`${name} has an unsupported format`); + return normalized; +} + +function faultScope(options: FaultOptions): string { + return createHash("sha256").update(JSON.stringify({ + level: options.level, + group: options.group, + account: options.account, + model: options.model, + stream: options.stream, + endpoint: options.endpoint, + requestId: options.requestId, + targetId: options.targetId, + })).digest("hex").slice(0, 16); +} + +function encodePageToken(offset: number, scope: string): string { + return Buffer.from(JSON.stringify({ v: 1, offset, scope }), "utf8").toString("base64url"); +} + +function decodePageToken(token: string | null, scope: string): number { + if (token === null) return 0; + try { + const parsed = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as Record; + if (parsed.v !== 1 || parsed.scope !== scope || !Number.isInteger(parsed.offset) || Number(parsed.offset) < 0) throw new Error("invalid"); + return Number(parsed.offset); + } catch { + throw new Error("--page-token is invalid or belongs to different filters"); + } +} + +function renderFaults(response: Record): RenderedCliResult { + const faults = response.faults && typeof response.faults === "object" ? response.faults as Record : {}; + const lines = ["PLATFORM-INFRA SUB2API FAULTS"]; + const window = record(faults.window); + lines.push(`WINDOW ${text(window.timeRange)} SOURCE native-admin-ops PROJECTION unidesk-cli`); + lines.push(""); + const summary = arrayOfRecords(faults.summary); + if (summary.length > 0) { + lines.push("GROUPS"); + lines.push(table(["GROUP", "P0", "CUSTOMER_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERR%", "ABSORB%"], summary.map((row) => [ + `${text(row.groupName)} (${text(row.groupId)})`, + text(row.p0), percent(row.customerErrorRatePercent), + text(row.p1), milliseconds(row.ttftP99Ms), + text(row.p2), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent), + ]))); + } + const details = arrayOfRecords(faults.details); + if (details.length > 0) { + lines.push("", `DETAILS ${text(faults.level ?? "SUMMARY")}`); + const columns = Array.isArray(faults.detailColumns) ? faults.detailColumns.map(String) : []; + lines.push(table(columns, details.map((row) => columns.map((column) => text(row[column]))))); + } + const boundary = record(faults.boundary); + lines.push("", "BOUNDARY"); + for (const value of Array.isArray(boundary.notes) ? boundary.notes : []) lines.push(`- ${String(value)}`); + const pagination = record(faults.pagination); + if (pagination.nextPageToken) lines.push("", `NEXT_PAGE_TOKEN ${text(pagination.nextPageToken)}`); + if (faults.next) lines.push("", "NEXT", ...String(faults.next).split("\n").map((line) => ` ${line}`)); + return { + ok: response.ok === true, + command: "platform-infra sub2api codex-pool faults", + renderedText: lines.join("\n"), + contentType: "text/plain", + projection: response, + }; +} + +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function arrayOfRecords(value: unknown): Record[] { + return Array.isArray(value) ? value.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; +} + +function text(value: unknown): string { + if (value === null || value === undefined || value === "") return "-"; + if (typeof value === "boolean") return value ? "yes" : "no"; + return String(value).replace(/[\r\n\t]+/gu, " "); +} + +function percent(value: unknown): string { + return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(2)}%` : text(value); +} + +function milliseconds(value: unknown): string { + return typeof value === "number" && Number.isFinite(value) ? `${Math.round(value)}ms` : text(value); +} + +function table(headers: string[], rows: string[][]): string { + const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length))); + return [headers, ...rows].map((row) => row.map((cell, index) => cell.padEnd(widths[index]!)).join(" ").trimEnd()).join("\n"); +} + +function pyJson(value: unknown): string { + return `json.loads(${JSON.stringify(JSON.stringify(value))})`; +} + +function faultScript(payload: unknown, target: ReturnType): string { + return ` +set -u +python3 - <<'PY' +import base64 +import json +import math +import subprocess +import sys +from urllib.parse import urlencode + +RUNTIME_MODE = ${pyJson(target.runtimeMode)} +NAMESPACE = ${pyJson(target.namespace)} +HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)} +APP_SECRET_NAME = ${pyJson(target.appSecretName)} +PAYLOAD = ${pyJson(payload)} +APP_POD = None + +def run(cmd, input_bytes=None): + return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + +def docker(args): + proc = run(["docker", *args]) + if proc.returncode == 0: + return proc + sudo_proc = run(["sudo", "-n", "docker", *args]) + return sudo_proc if sudo_proc.returncode == 0 else proc + +def kubectl(args, input_bytes=None): + return run(["kubectl", *args], input_bytes) + +def kube_json(args, label): + proc = kubectl([*args, "-o", "json"]) + if proc.returncode != 0: + raise RuntimeError(label + " failed") + return json.loads(proc.stdout.decode("utf-8")) + +if RUNTIME_MODE != "host-docker": + pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app=sub2api"], "list pods").get("items") or [] + ready = [item for item in pods if item.get("status", {}).get("phase") == "Running"] + if not ready: + raise RuntimeError("sub2api app pod not found") + APP_POD = ready[0]["metadata"]["name"] + +def config_value(name, key, default=None): + if RUNTIME_MODE == "host-docker": + proc = docker(["inspect", "sub2api-app", "--format", "{{json .Config.Env}}"]) + if proc.returncode != 0: + return default + for item in json.loads(proc.stdout.decode("utf-8")): + if item.startswith(key + "="): + return item.split("=", 1)[1] + return default + data = kube_json(["-n", NAMESPACE, "get", "configmap", name], "configmap/" + name).get("data") or {} + return data.get(key, default) + +def secret_value(name, key): + if RUNTIME_MODE == "host-docker": + return config_value("", key) + data = kube_json(["-n", NAMESPACE, "get", "secret", name], "secret/" + name).get("data") or {} + return base64.b64decode(data[key]).decode("utf-8") if key in data else None + +def curl_api(method, path, bearer=None, payload=None): + body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8") + script = r'''set -eu +method="$1"; url="$2"; token="\${3:-}"; tmp="$(mktemp)"; trap 'rm -f "$tmp"' EXIT; cat > "$tmp" +args=""; [ -n "$token" ] && args="Authorization: Bearer $token" +if [ -n "$args" ] && [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url" +elif [ -n "$args" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" "$url" +elif [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url" +else curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" "$url"; fi''' + base = f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}" if RUNTIME_MODE == "host-docker" else "http://127.0.0.1:8080" + cmd = ["sh", "-c", script, "sh", method, base + path, bearer or ""] + proc = run(cmd, body) if RUNTIME_MODE == "host-docker" else run(["kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, "--", *cmd], body) + output = proc.stdout.decode("utf-8", errors="replace") + marker = "\\n__HTTP_CODE__:" + pos = output.rfind(marker) + status = int(output[pos + len(marker):].strip()[-3:]) if pos >= 0 else 0 + raw = output[:pos] if pos >= 0 else output + try: + parsed = json.loads(raw) if raw.strip() else None + except json.JSONDecodeError: + parsed = None + return {"ok": proc.returncode == 0 and 200 <= status < 300, "status": status, "json": parsed, "body": raw[:300]} + +def data_of(response, label): + parsed = response.get("json") + code = parsed.get("code") if isinstance(parsed, dict) else None + if response.get("ok") is not True or (code is not None and code != 0): + message = parsed.get("message") if isinstance(parsed, dict) else response.get("body") + raise RuntimeError(f"{label} failed: http={response.get('status')} message={message}") + return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed + +def login(): + email = config_value("sub2api-config", "ADMIN_EMAIL", "admin@example.com") + password = secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD") + if not password: + raise RuntimeError("ADMIN_PASSWORD missing") + data = data_of(curl_api("POST", "/api/v1/auth/login", payload={"email": email, "password": password}), "admin login") + token = data.get("access_token") or data.get("token") if isinstance(data, dict) else None + if not token: + raise RuntimeError("admin login response has no token") + return token + +def get(token, path, params=None, label=None): + suffix = "?" + urlencode({key: value for key, value in (params or {}).items() if value is not None}) if params else "" + return data_of(curl_api("GET", path + suffix, bearer=token), label or path) + +def items_of(data): + if isinstance(data, list): + return data + if isinstance(data, dict): + for key in ("items", "groups", "accounts"): + if isinstance(data.get(key), list): + return data[key] + return [] + +def total_of(data): + return int(data.get("total", len(items_of(data)))) if isinstance(data, dict) else len(items_of(data)) + +def percent(value): + if not isinstance(value, (int, float)): + return None + return round(value * 100 if value <= 1 else value, 4) + +def selector_id(value): + return int(value) if isinstance(value, str) and value.isdigit() else None + +def filters_for(group_id=None): + filters = PAYLOAD["filters"] + result = {"time_range": "24h", "platform": "openai", "group_id": group_id} + account = filters.get("account") + if account: + result["account_id" if selector_id(account) is not None else "account_name"] = selector_id(account) if selector_id(account) is not None else account + if filters.get("model"): + result["model"] = filters["model"] + if filters.get("stream"): + result["stream"] = "true" if filters["stream"] == "stream" else "false" + if filters.get("endpoint"): + result["inbound_endpoint"] = filters["endpoint"] + if filters.get("requestId"): + result["request_id"] = filters["requestId"] + return result + +def selected_groups(groups): + selector = PAYLOAD["filters"].get("group") + if not selector: + return groups + selected = [item for item in groups if str(item.get("id")) == str(selector) or item.get("name") == selector] + if len(selected) != 1: + raise RuntimeError("group-not-found-or-ambiguous: " + str(selector)) + return selected + +def threshold_state(value, threshold): + if not isinstance(value, (int, float)) or not isinstance(threshold, (int, float)): + return "unavailable" + return "breach" if value >= threshold else "ok" + +def availability_by_account(token, group_id): + data = get(token, "/api/v1/admin/ops/account-availability", {"platform": "openai", "group_id": group_id}, "account availability") + raw = data.get("account") if isinstance(data, dict) else None + return raw if isinstance(raw, dict) else {} + +def detail_row(item, group_metrics, availability, level, absorbed=None): + account_id = item.get("account_id") + state = availability.get(str(account_id), {}) if isinstance(availability, dict) else {} + endpoint = item.get("inbound_endpoint") or item.get("request_path") or item.get("path") + base = { + "GROUP": f"{item.get('group_name') or group_metrics.get('groupName')} ({item.get('group_id') or group_metrics.get('groupId')})", + "ACCOUNT": f"{item.get('account_name') or '-'} ({account_id or '-'})", + "MODEL": item.get("requested_model") or item.get("model") or "-", + "MODE": "stream" if item.get("stream") is True else "sync" if item.get("stream") is False else "-", + "ENDPOINT": endpoint or "-", + "STATUS": item.get("status_code") or "-", + "REQUEST_ID": item.get("request_id") or item.get("client_request_id") or "-", + } + if level == "P0": + base.update({"CUSTOMER_ERR%": group_metrics.get("customerErrorRatePercent"), "ABSORBED": False}) + else: + unavailable_reason = str(state.get("unavailable_reason") or state.get("reason") or "").lower() + if state.get("is_available") is False and ("temporary" in unavailable_reason or "cooldown" in unavailable_reason): + temp_unsched = "active" + elif state.get("is_available") is True: + temp_unsched = "none" + else: + temp_unsched = "unavailable" + base.update({"ROOT": item.get("error_source") or item.get("message") or "upstream", "TEMP_UNSCHED": temp_unsched, "ABSORBED": absorbed}) + return base + +def execute(): + token = login() + thresholds = get(token, "/api/v1/admin/ops/settings/metric-thresholds", label="metric thresholds") + groups = selected_groups(items_of(get(token, "/api/v1/admin/groups/all", {"platform": "openai"}, "list groups"))) + summaries = [] + group_data = [] + for group in groups: + group_id = group.get("id") + native = get(token, "/api/v1/admin/ops/dashboard/overview", {"time_range": "24h", "platform": "openai", "group_id": group_id}, "dashboard overview") + request_errors = get(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors") + upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors") + customer_total = total_of(request_errors) + upstream_total = total_of(upstream_errors) + absorbed = max(0, upstream_total - customer_total) + absorbed_percent = round(absorbed * 100 / upstream_total, 2) if upstream_total else None + customer_rate = percent(native.get("error_rate") if isinstance(native, dict) else None) + upstream_rate = percent(native.get("upstream_error_rate") if isinstance(native, dict) else None) + ttft = native.get("ttft") if isinstance(native, dict) and isinstance(native.get("ttft"), dict) else {} + metrics = { + "groupId": group_id, + "groupName": group.get("name"), + "customerErrorCount": customer_total, + "customerErrorRatePercent": customer_rate, + "upstreamErrorCount": upstream_total, + "upstreamErrorRatePercent": upstream_rate, + "absorbedCountProjection": absorbed, + "absorbedPercent": absorbed_percent, + "ttftP99Ms": ttft.get("p99_ms"), + } + metrics["p0"] = "active" if customer_total > 0 else "clear" + metrics["p1"] = threshold_state(metrics["ttftP99Ms"], thresholds.get("ttft_p99_ms_max")) + metrics["p2"] = threshold_state(upstream_rate, thresholds.get("upstream_error_rate_percent_max")) + summaries.append(metrics) + group_data.append((group, metrics)) + level = PAYLOAD["filters"].get("level") + offset = int(PAYLOAD.get("offset") or 0) + page = offset // int(PAYLOAD["pageSize"]) + 1 + details = [] + detail_columns = [] + total = 0 + if level in ("P0", "P2"): + for group, metrics in group_data: + group_id = group.get("id") + endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors" + data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") + total += total_of(data) + availability = availability_by_account(token, group_id) if level == "P2" else {} + for item in items_of(data): + absorbed = None + if level == "P2" and item.get("request_id"): + correlated = get(token, "/api/v1/admin/ops/request-errors", {"time_range": "24h", "request_id": item.get("request_id"), "page": 1, "page_size": 1, "view": "all"}, "correlated request error") + absorbed = total_of(correlated) == 0 + details.append(detail_row(item, metrics, availability, level, absorbed)) + detail_columns = ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "CUSTOMER_ERR%", "ABSORBED", "REQUEST_ID"] if level == "P0" else ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "ROOT", "TEMP_UNSCHED", "ABSORBED", "REQUEST_ID"] + elif level == "P1": + total = len(group_data) * 2 + endpoint_names = ["/v1/responses", "/v1/responses/compact"] + for group, metrics in group_data: + for endpoint in endpoint_names: + if PAYLOAD["filters"].get("endpoint") and PAYLOAD["filters"].get("endpoint") != endpoint: + continue + details.append({ + "GROUP": f"{metrics.get('groupName')} ({metrics.get('groupId')})", + "ENDPOINT": endpoint, + "SAMPLES": "unavailable", + "P50": "unavailable", + "P95": "unavailable", + "P99": "unavailable", + "MAX": "unavailable", + "OUTLIER_REQUEST_ID": "unavailable", + "BOUNDARY": "native endpoint/request TTFT unavailable; total duration not substituted", + }) + detail_columns = ["GROUP", "ENDPOINT", "SAMPLES", "P50", "P95", "P99", "MAX", "OUTLIER_REQUEST_ID", "BOUNDARY"] + return { + "ok": True, + "level": level, + "window": {"timeRange": "24h"}, + "thresholds": thresholds, + "summary": summaries, + "details": details[:PAYLOAD["pageSize"]], + "detailColumns": detail_columns, + "pagination": {"offset": offset, "total": total, "hasMore": offset + PAYLOAD["pageSize"] < total}, + "boundary": {"notes": [ + "P0/P2 facts come from native admin ops request-errors, upstream-errors, overview, and account-availability.", + "P0/P1/P2 labels and failover/retry absorption are UniDesk CLI projections; Sub2API native severity is not rewritten.", + "P1 endpoint-level samples and request-level TTFT are unavailable in the observed native Ops response; total duration is never used as TTFT.", + "Absorption summary is a bounded count projection from native upstream/customer totals; P2 detail absorption is correlated by stable request ID.", + ]}, + "next": "Use --level P0|P1|P2 for details; reuse filters with --page-token for the next fixed page; use codex-pool trace --request-id for trace disclosure.", + "valuesPrinted": False, + } + +try: + output = execute() +except Exception as exc: + output = {"ok": False, "error": str(exc), "valuesPrinted": False} +print(json.dumps(output, ensure_ascii=False, indent=2)) +sys.exit(0 if output.get("ok") else 1) +PY +`; +} diff --git a/scripts/src/platform-infra-sub2api-codex/options.ts b/scripts/src/platform-infra-sub2api-codex/options.ts index a17ddade..64ce9e00 100644 --- a/scripts/src/platform-infra-sub2api-codex/options.ts +++ b/scripts/src/platform-infra-sub2api-codex/options.ts @@ -25,6 +25,7 @@ import type { ConfirmOptions, DisclosureOptions, SentinelImageOptions, SentinelP import { codexPoolCleanupProbes, codexPoolConfigureLocal, codexPoolExpose, codexPoolPlan, codexPoolSentinelImage, codexPoolSentinelProbe, codexPoolSentinelReport, codexPoolSync, codexPoolTrace, codexPoolValidate } from "./actions"; import { renderCodexPoolPlan } from "./render"; import { codexPoolRuntime } from "./runtime"; +import { codexPoolFaults } from "./faults"; import { defaultCodexPoolRuntimeTargetId } from "./runtime-target"; import { codexPoolHelp } from "./types"; @@ -38,6 +39,7 @@ export async function runCodexPoolCommand(config: UniDeskConfig, args: string[]) if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1))); if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1))); if (action === "runtime") return await codexPoolRuntime(config, args.slice(1)); + if (action === "faults") return await codexPoolFaults(config, args.slice(1)); if (action === "trace") return await codexPoolTrace(config, parseTraceOptions(args.slice(1))); if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1))); if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1))); diff --git a/scripts/src/platform-infra-sub2api-codex/types.ts b/scripts/src/platform-infra-sub2api-codex/types.ts index 347d4685..5c54219c 100644 --- a/scripts/src/platform-infra-sub2api-codex/types.ts +++ b/scripts/src/platform-infra-sub2api-codex/types.ts @@ -341,7 +341,7 @@ export function codexPoolHelp(): unknown { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(); return { - command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local", + command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|faults|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local", output: "json, except trace and sentinel-report default to low-noise text tables", usage: [ "bun scripts/cli.ts platform-infra sub2api codex-pool plan", @@ -351,6 +351,7 @@ export function codexPoolHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account [--target PK01] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--account ] [--since 24h] [--tail 50000] [--target PK01] [--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool faults [--level P0|P1|P2] [--group ] [--account ] [--model ] [--stream sync|stream] [--endpoint ] [--request-id ] [--page-token ] [--target PK01] [--json]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --account --template [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --account --kind temp-unschedulable [--target PK01] [--confirm]", "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]", From 0affaa4827c761a9987e4c081c87bd78d86615d2 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:42:05 +0200 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20=E5=85=BC=E5=AE=B9=20Sub2API=200.1.1?= =?UTF-8?q?47=20=E6=95=85=E9=9A=9C=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform-infra-sub2api-codex/faults.ts | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/scripts/src/platform-infra-sub2api-codex/faults.ts b/scripts/src/platform-infra-sub2api-codex/faults.ts index 8f89e410..61e276bd 100644 --- a/scripts/src/platform-infra-sub2api-codex/faults.ts +++ b/scripts/src/platform-infra-sub2api-codex/faults.ts @@ -397,6 +397,19 @@ def items_of(data): def total_of(data): return int(data.get("total", len(items_of(data)))) if isinstance(data, dict) else len(items_of(data)) +def all_items(token, path, params, label): + page = 1 + page_size = 500 + rows = [] + while True: + data = get(token, path, {**params, "page": page, "page_size": page_size}, label) + batch = items_of(data) + rows.extend(batch) + total = total_of(data) + if not batch or len(rows) >= total: + return rows + page += 1 + def percent(value): if not isinstance(value, (int, float)): return None @@ -417,8 +430,6 @@ def filters_for(group_id=None): result["stream"] = "true" if filters["stream"] == "stream" else "false" if filters.get("endpoint"): result["inbound_endpoint"] = filters["endpoint"] - if filters.get("requestId"): - result["request_id"] = filters["requestId"] return result def selected_groups(groups): @@ -510,14 +521,30 @@ def execute(): for group, metrics in group_data: group_id = group.get("id") endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors" - data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") - total += total_of(data) + request_id = PAYLOAD["filters"].get("requestId") + if request_id: + rows = [ + item for item in all_items(token, endpoint, {**filters_for(group_id), "view": "all", "include_detail": 1}, level + " request lookup") + if item.get("request_id") == request_id or item.get("client_request_id") == request_id + ] + total += len(rows) + rows = rows[offset:offset + PAYLOAD["pageSize"]] + else: + data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") + total += total_of(data) + rows = items_of(data) availability = availability_by_account(token, group_id) if level == "P2" else {} - for item in items_of(data): + customer_request_ids = set() + if level == "P2": + customer_request_ids = { + item.get("request_id") or item.get("client_request_id") + for item in all_items(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "view": "all"}, "customer error correlation") + if item.get("request_id") or item.get("client_request_id") + } + for item in rows: absorbed = None if level == "P2" and item.get("request_id"): - correlated = get(token, "/api/v1/admin/ops/request-errors", {"time_range": "24h", "request_id": item.get("request_id"), "page": 1, "page_size": 1, "view": "all"}, "correlated request error") - absorbed = total_of(correlated) == 0 + absorbed = item.get("request_id") not in customer_request_ids details.append(detail_row(item, metrics, availability, level, absorbed)) detail_columns = ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "CUSTOMER_ERR%", "ABSORBED", "REQUEST_ID"] if level == "P0" else ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "ROOT", "TEMP_UNSCHED", "ABSORBED", "REQUEST_ID"] elif level == "P1": From cfba88f0c0677041fca0b968aa855b98237aa766 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 14 Jul 2026 12:44:34 +0200 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=E7=B2=BE=E7=A1=AE=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=20Sub2API=200.1.147=20=E6=95=85=E9=9A=9C=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform-infra-sub2api-codex/faults.ts | 72 ++++++++++++------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/scripts/src/platform-infra-sub2api-codex/faults.ts b/scripts/src/platform-infra-sub2api-codex/faults.ts index 61e276bd..57e72890 100644 --- a/scripts/src/platform-infra-sub2api-codex/faults.ts +++ b/scripts/src/platform-infra-sub2api-codex/faults.ts @@ -218,11 +218,11 @@ function renderFaults(response: Record): RenderedCliResult { const summary = arrayOfRecords(faults.summary); if (summary.length > 0) { lines.push("GROUPS"); - lines.push(table(["GROUP", "P0", "CUSTOMER_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERR%", "ABSORB%"], summary.map((row) => [ + lines.push(table(["GROUP", "P0", "CUSTOMER_ERRS", "GROUP_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERRS", "GROUP_UP_ERR%", "ABSORB%"], summary.map((row) => [ `${text(row.groupName)} (${text(row.groupId)})`, - text(row.p0), percent(row.customerErrorRatePercent), + text(row.p0), text(row.customerErrorCount), percent(row.customerErrorRatePercent), text(row.p1), milliseconds(row.ttftP99Ms), - text(row.p2), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent), + text(row.p2), text(row.upstreamErrorCount), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent), ]))); } const details = arrayOfRecords(faults.details); @@ -418,19 +418,43 @@ def percent(value): def selector_id(value): return int(value) if isinstance(value, str) and value.isdigit() else None -def filters_for(group_id=None): - filters = PAYLOAD["filters"] +def scope_filters(group_id=None): result = {"time_range": "24h", "platform": "openai", "group_id": group_id} + return result + +def matches_detail_filters(item): + filters = PAYLOAD["filters"] account = filters.get("account") if account: - result["account_id" if selector_id(account) is not None else "account_name"] = selector_id(account) if selector_id(account) is not None else account - if filters.get("model"): - result["model"] = filters["model"] + account_match = str(item.get("account_id") or "") == str(account) or str(item.get("account_name") or "") == str(account) + if not account_match: + return False + if filters.get("model") and str(item.get("requested_model") or item.get("model") or "") != str(filters["model"]): + return False if filters.get("stream"): - result["stream"] = "true" if filters["stream"] == "stream" else "false" - if filters.get("endpoint"): - result["inbound_endpoint"] = filters["endpoint"] - return result + expected_stream = filters["stream"] == "stream" + if item.get("stream") is not expected_stream: + return False + if filters.get("endpoint") and str(item.get("inbound_endpoint") or item.get("request_path") or item.get("path") or "") != str(filters["endpoint"]): + return False + if filters.get("requestId"): + request_id = filters["requestId"] + if item.get("request_id") != request_id and item.get("client_request_id") != request_id: + return False + return True + +def has_detail_filters(): + filters = PAYLOAD["filters"] + return any(filters.get(key) for key in ("account", "model", "stream", "endpoint", "requestId")) + +DETAIL_ROWS = {} + +def error_rows(token, path, group_id, apply_filters): + cache_key = (path, group_id, apply_filters) + if cache_key not in DETAIL_ROWS: + rows = all_items(token, path, {**scope_filters(group_id), "view": "all", "include_detail": 1}, "fault detail scan") + DETAIL_ROWS[cache_key] = [item for item in rows if matches_detail_filters(item)] if apply_filters else rows + return DETAIL_ROWS[cache_key] def selected_groups(groups): selector = PAYLOAD["filters"].get("group") @@ -486,10 +510,14 @@ def execute(): for group in groups: group_id = group.get("id") native = get(token, "/api/v1/admin/ops/dashboard/overview", {"time_range": "24h", "platform": "openai", "group_id": group_id}, "dashboard overview") - request_errors = get(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors") - upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors") - customer_total = total_of(request_errors) - upstream_total = total_of(upstream_errors) + if has_detail_filters(): + customer_total = len(error_rows(token, "/api/v1/admin/ops/request-errors", group_id, True)) + upstream_total = len(error_rows(token, "/api/v1/admin/ops/upstream-errors", group_id, True)) + else: + request_errors = get(token, "/api/v1/admin/ops/request-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors") + upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors") + customer_total = total_of(request_errors) + upstream_total = total_of(upstream_errors) absorbed = max(0, upstream_total - customer_total) absorbed_percent = round(absorbed * 100 / upstream_total, 2) if upstream_total else None customer_rate = percent(native.get("error_rate") if isinstance(native, dict) else None) @@ -521,16 +549,12 @@ def execute(): for group, metrics in group_data: group_id = group.get("id") endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors" - request_id = PAYLOAD["filters"].get("requestId") - if request_id: - rows = [ - item for item in all_items(token, endpoint, {**filters_for(group_id), "view": "all", "include_detail": 1}, level + " request lookup") - if item.get("request_id") == request_id or item.get("client_request_id") == request_id - ] + if has_detail_filters(): + rows = error_rows(token, endpoint, group_id, True) total += len(rows) rows = rows[offset:offset + PAYLOAD["pageSize"]] else: - data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") + data = get(token, endpoint, {**scope_filters(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") total += total_of(data) rows = items_of(data) availability = availability_by_account(token, group_id) if level == "P2" else {} @@ -538,7 +562,7 @@ def execute(): if level == "P2": customer_request_ids = { item.get("request_id") or item.get("client_request_id") - for item in all_items(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "view": "all"}, "customer error correlation") + for item in error_rows(token, "/api/v1/admin/ops/request-errors", group_id, False) if item.get("request_id") or item.get("client_request_id") } for item in rows: