diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index 3e0d09e4..d253af77 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -10,6 +10,8 @@ description: >- UniDesk 通过 `platform-infra sub2api` 运维 YAML 选中的 Sub2API target；当前 active target 以 `config/platform-infra/sub2api.yaml` 为准，可能是 PK01 host-Docker 或 k3s target。日常操作统一使用 UniDesk CLI，不直接写 Kubernetes 资源、不手工调用 Sub2API 管理 API、不打印 Secret。 +本技能遵循 `Skill(cli-spec)`。默认输出使用紧凑文本表格，机器处理显式使用 `--json`，大结果使用固定分页和精确下钻。 + ## 高频入口 ```bash diff --git a/.agents/skills/unidesk-sub2api/references/codex-pool.md b/.agents/skills/unidesk-sub2api/references/codex-pool.md index 4b17aee4..eb09b7b0 100644 --- a/.agents/skills/unidesk-sub2api/references/codex-pool.md +++ b/.agents/skills/unidesk-sub2api/references/codex-pool.md @@ -24,6 +24,8 @@ - 默认只给有界 request ID 索引，使用 `trace` 做精确下钻； - 指定分组和 `--account` 下钻提供上游账号质量评分： - 成功请求分母和账号级 TTFT 来自原生 admin usage； + - 同一张账号表必须聚合请求数、Token、标准 API 美元计费基数、账号名称末尾的人民币采购倍率和人民币上游成本； + - 常规评分与成本评估只运行一次 `runtime errors --group ... --since ...`，不再分别调用 usage、profit 和逐账号查询拼表； - 客户错误来自原生 request-errors； - 切号、同账号重试、临时不可调度和 `forward_failed` 来自原生 system-log 索引； - 默认权重为可靠性 60、TTFT 25、当前可用性 15，并同时输出评分组成、等级、置信度和扣分原因； @@ -33,6 +35,15 @@ - 请求证据不足时使用低置信度，不能据此自动调整 priority、capacity 或 schedulable； - 账号名称是人类可读运维标识，应与账号 ID 一起完整输出；API Key、token 和 credentials 仍必须脱敏； - 只有带 `group_id` 的 failover 和 `forward_failed` request ID 才进入分组账号失败率；账号级临时不可调度和同账号重试可以作为共享运维信号展示，但不加入分组失败分子； + - 客户错误只有满足以下任一条件时才计入可靠性扣分： + - 原生阶段或错误分类明确标记为 upstream； + - 已选择账号且消息命中稳定的上游网关故障语义； + - 以下错误必须显示在原始客户错误和排除原因中，但不扣账号分： + - 上下文窗口超限和客户输入校验； + - 模型路由或分组模型配置错误； + - internal、client 或 business 阶段错误； + - 没有选中账号或无法确认上游归属的客户错误； + - 排除项必须按 request ID 去重，并输出 `scoreableUpstreamErrorRequests`、`excludedNonUpstreamErrorRequests` 和原因桶；禁止把未知归因默认算作上游失败； - 账号被多个分组共享时，账号级 upstream-errors 和缺少 `group_id` 的 runtime 事件属于共享证据，禁止跨分组相加； - dashboard overview 提供请求分母和错误率； - dashboard overview 的 `ttft` 提供成功流式请求的首 token 分位数，`duration` 提供成功请求总耗时分位数；二者不能互相替代； diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-head.ts b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-head.ts new file mode 100644 index 00000000..fb9f6022 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-head.ts @@ -0,0 +1,1089 @@ +export interface RuntimeRemoteTarget { + runtimeMode: string; + namespace: string; + hostDockerAppPort: number | null; + hostDockerEnvPath: string | null; + appSecretName: string; +} + +function pyJson(value: unknown): string { + return `json.loads(${JSON.stringify(JSON.stringify(value))})`; +} + +export function runtimeRemoteScriptHead(payload: unknown, target: RuntimeRemoteTarget): string { + return ` +set -u +python3 - <<'PY' +import json +import re +import socket +import struct +import subprocess +import sys +from datetime import datetime, timedelta, timezone +from urllib.parse import quote, urlsplit + +RUNTIME_MODE = ${pyJson(target.runtimeMode)} +NAMESPACE = ${pyJson(target.namespace)} +HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)} +HOST_DOCKER_ENV_PATH = ${pyJson(target.hostDockerEnvPath)} +HOST_DOCKER_APP_CONTAINER = "sub2api-app" +APP_SECRET_NAME = ${pyJson(target.appSecretName)} +PAYLOAD = ${pyJson(payload)} + +def run(cmd, input_bytes=None): + return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + +def text(value, limit=1000): + if isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + return str(value)[-limit:] + +def docker(args): + proc = run(["docker", *args]) + if proc.returncode == 0: + return proc + sudo_proc = run(["sudo", "-n", "docker", *args]) + return sudo_proc if sudo_proc.returncode == 0 else proc + +def kubectl(args, input_bytes=None): + return run(["kubectl", *args], input_bytes) + +def kube_json(args, label): + proc = kubectl([*args, "-o", "json"]) + if proc.returncode != 0: + raise RuntimeError(f"{label} failed: {text(proc.stderr)}") + return json.loads(proc.stdout.decode("utf-8")) + +def read_host_env(): + try: + with open(HOST_DOCKER_ENV_PATH, "r", encoding="utf-8") as handle: + lines = handle.read().splitlines() + except PermissionError: + proc = run(["sudo", "-n", "cat", HOST_DOCKER_ENV_PATH]) + if proc.returncode != 0: + raise RuntimeError("read host env failed: " + text(proc.stderr)) + lines = proc.stdout.decode("utf-8", errors="replace").splitlines() + values = {} + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, value = stripped.split("=", 1) + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + value = value[1:-1] + values[key.strip()] = value + return values + +def select_app_pod(): + if RUNTIME_MODE == "host-docker": + return HOST_DOCKER_APP_CONTAINER + pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api"], "sub2api pods").get("items") or [] + running = [item for item in pods if (item.get("status") or {}).get("phase") == "Running"] + if not running: + raise RuntimeError("sub2api app pod not found") + return running[0]["metadata"]["name"] + +APP_POD = select_app_pod() + +def config_value(name, key, fallback=None): + if RUNTIME_MODE == "host-docker": + return read_host_env().get(key) or fallback + data = kube_json(["-n", NAMESPACE, "get", "configmap", name], f"configmap/{name}").get("data") or {} + return data.get(key) or fallback + +def secret_value(name, key): + if RUNTIME_MODE == "host-docker": + return read_host_env().get(key) + import base64 + data = kube_json(["-n", NAMESPACE, "get", "secret", name], f"secret/{name}").get("data") or {} + return base64.b64decode(data[key]).decode("utf-8") if key in data else None + +def parse_curl(proc): + output = proc.stdout.decode("utf-8", errors="replace") + marker = "\\n__HTTP_CODE__:" + pos = output.rfind(marker) + if pos < 0: + return {"ok": False, "httpStatus": 0, "json": None, "body": output, "stderr": text(proc.stderr)} + body = output[:pos] + try: + status = int(output[pos + len(marker):].strip()[-3:]) + except ValueError: + status = 0 + try: + parsed = json.loads(body) if body.strip() else None + except json.JSONDecodeError: + parsed = None + return {"ok": proc.returncode == 0 and 200 <= status < 300, "httpStatus": status, "json": parsed, "body": body, "stderr": text(proc.stderr)} + +def curl_api(method, path, bearer=None, payload=None): + body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8") + script = r''' +set -eu +method="$1" +url="$2" +token="\${3:-}" +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT +cat > "$tmp" +if [ -n "$token" ] && [ "$method" = GET ] && [ ! -s "$tmp" ]; then + curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "Authorization: Bearer $token" "$url" +elif [ -n "$token" ]; then + curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "Authorization: Bearer $token" --data-binary @"$tmp" "$url" +elif [ "$method" = GET ] && [ ! -s "$tmp" ]; then + curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" "$url" +else + curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url" +fi +''' + if RUNTIME_MODE == "host-docker": + proc = run(["sh", "-c", script, "sh", method, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}{path}", bearer or ""], body) + else: + proc = run(["kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or ""], body) + return parse_curl(proc) + +def data_of(response, label): + parsed = response.get("json") + code = parsed.get("code") if isinstance(parsed, dict) else None + if response.get("ok") is not True or (code is not None and code != 0): + message = parsed.get("message") if isinstance(parsed, dict) else text(response.get("body"), 500) + raise RuntimeError(f"{label} failed: http={response.get('httpStatus')} message={message}") + return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed + +def items_of(data): + if isinstance(data, list): + return data + if isinstance(data, dict): + for key in ("items", "groups", "accounts"): + if isinstance(data.get(key), list): + return data[key] + return [] + +def login(): + email = config_value("sub2api-config", "ADMIN_EMAIL", PAYLOAD["adminEmailDefault"]) + password = secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD") + if not password: + raise RuntimeError("ADMIN_PASSWORD missing") + data = data_of(curl_api("POST", "/api/v1/auth/login", payload={"email": email, "password": password}), "admin login") + token = None + if isinstance(data, dict): + token = data.get("access_token") or data.get("token") + if not token: + raise RuntimeError("admin login response has no token") + return token + +def list_groups(token, platform=None): + suffix = "?platform=" + quote(str(platform), safe="") if platform else "" + return items_of(data_of(curl_api("GET", "/api/v1/admin/groups/all" + suffix, bearer=token), "list groups")) + +def list_group_accounts(token, group_id, platform=None): + path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500" + if platform: + path += "&platform=" + quote(str(platform), safe="") + return items_of(data_of(curl_api("GET", path, bearer=token), "list group accounts")) + +def account_detail(token, account_id): + return data_of(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), "get account") + +def normalize_policy(credentials): + if not isinstance(credentials, dict): + credentials = {} + enabled = credentials.get("temp_unschedulable_enabled") is True + raw = credentials.get("temp_unschedulable_rules") + if isinstance(raw, str): + try: + raw = json.loads(raw) + except json.JSONDecodeError: + raw = [] + rules = raw if isinstance(raw, list) else [] + codes = sorted(set(item.get("error_code") for item in rules if isinstance(item, dict) and isinstance(item.get("error_code"), int))) + return {"enabled": enabled, "ruleCount": len(rules), "statusCodes": codes, "rules": rules} + +def template_summary(template): + policy = normalize_policy(template.get("credentials")) + return {"id": template.get("id"), "kind": template.get("kind"), "description": template.get("description"), **{key: policy[key] for key in ("enabled", "ruleCount", "statusCodes")}} + +def matching_template(credentials): + current = normalize_policy(credentials) + for template in PAYLOAD["templates"]: + if normalize_policy(template.get("credentials")) == current: + return template.get("id") + return None + +def account_summary(detail, include_rules=False): + credentials = detail.get("credentials") if isinstance(detail.get("credentials"), dict) else {} + policy = normalize_policy(credentials) + name = detail.get("name") + management = "yaml-managed" if name in PAYLOAD["managedAccountNames"] else ("yaml-protected-manual" if name in PAYLOAD["protectedManualAccountNames"] else "runtime-manual") + result = { + "accountName": name, + "accountId": detail.get("id"), + "management": management, + "type": detail.get("type"), + "status": detail.get("status"), + "schedulable": detail.get("schedulable"), + "proxyId": detail.get("proxy_id"), + "concurrencyLimit": detail.get("concurrency"), + "currentConcurrency": detail.get("current_concurrency"), + "loadFactor": detail.get("load_factor"), + "priority": detail.get("priority"), + "tempUnschedulable": {key: policy[key] for key in ("enabled", "ruleCount", "statusCodes")}, + "matchingTemplate": matching_template(credentials), + "poolModeEnabled": credentials.get("pool_mode") is True, + "credentialsPrinted": False, + } + if include_rules: + result["tempUnschedulable"]["rules"] = policy["rules"] + return result + +def list_proxies(token): + return items_of(data_of(curl_api("GET", "/api/v1/admin/proxies?page=1&page_size=500", bearer=token), "list proxies")) + +def proxy_summary(token, proxy_id): + if proxy_id is None: + return {"configured": False, "route": "direct", "credentialsPrinted": False} + matches = [item for item in list_proxies(token) if isinstance(item, dict) and item.get("id") == proxy_id] + if len(matches) != 1: + return {"configured": True, "id": proxy_id, "status": "record-not-found", "credentialsPrinted": False} + item = matches[0] + return { + "configured": True, + "route": "account-proxy", + "id": item.get("id"), + "name": item.get("name"), + "protocol": item.get("protocol"), + "host": item.get("host"), + "port": item.get("port"), + "status": item.get("status"), + "fallbackMode": item.get("fallback_mode"), + "lastCheckAt": item.get("last_check_at"), + "lastError": text(item.get("last_error"), 240) if item.get("last_error") else None, + "credentialsPrinted": False, + } + +def proxy_environment_summary(): + rows = [] + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + value = app_text(["sh", "-c", f"printenv {name} 2>/dev/null || true"]) + parsed = urlsplit(value) if value else None + rows.append({ + "name": name, + "present": bool(value), + "scheme": parsed.scheme if parsed else None, + "host": parsed.hostname if parsed else None, + "port": parsed.port if parsed else None, + "credentialsRedacted": bool(parsed and (parsed.username or parsed.password)), + }) + no_proxy = app_text(["sh", "-c", "printenv NO_PROXY 2>/dev/null || printenv no_proxy 2>/dev/null || true"]) + return rows, { + "present": bool(no_proxy), + "entryCount": len([item for item in no_proxy.split(",") if item.strip()]), + "hyueapiPresent": any(item.strip() in ("hyueapi.com", ".hyueapi.com") for item in no_proxy.split(",")), + "valuesPrinted": False, + } + +def app_runtime_summary(): + if RUNTIME_MODE == "host-docker": + proc = docker(["inspect", APP_POD]) + if proc.returncode != 0: + return {"runtimeMode": RUNTIME_MODE, "name": APP_POD, "status": "unavailable", "reason": text(proc.stderr, 240)} + values = json.loads(proc.stdout.decode("utf-8")) + item = values[0] if values else {} + state = item.get("State") or {} + host_config = item.get("HostConfig") or {} + networks = ((item.get("NetworkSettings") or {}).get("Networks") or {}) + return { + "runtimeMode": RUNTIME_MODE, + "name": APP_POD, + "state": state.get("Status"), + "health": ((state.get("Health") or {}).get("Status")), + "restartCount": item.get("RestartCount"), + "oomKilled": state.get("OOMKilled"), + "startedAt": state.get("StartedAt"), + "networkMode": host_config.get("NetworkMode"), + "networks": [ + {"name": name, "ipAddress": value.get("IPAddress"), "gateway": value.get("Gateway")} + for name, value in sorted(networks.items()) if isinstance(value, dict) + ], + } + pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], "sub2api pod") + spec = pod.get("spec") or {} + status = pod.get("status") or {} + restarts = sum(item.get("restartCount") or 0 for item in status.get("containerStatuses") or [] if isinstance(item, dict)) + return { + "runtimeMode": RUNTIME_MODE, + "name": APP_POD, + "state": status.get("phase"), + "restartCount": restarts, + "oomKilled": None, + "startedAt": status.get("startTime"), + "networkMode": "host" if spec.get("hostNetwork") is True else "pod", + "nodeName": spec.get("nodeName"), + "podIp": status.get("podIP"), + "dnsPolicy": spec.get("dnsPolicy"), + } + +def dns_summary(): + content = app_text(["cat", "/etc/resolv.conf"]) + nameservers = [] + search = [] + options = [] + for line in content.splitlines(): + fields = line.strip().split() + if not fields or fields[0].startswith("#"): + continue + if fields[0] == "nameserver": nameservers.extend(fields[1:]) + elif fields[0] == "search": search.extend(fields[1:]) + elif fields[0] == "options": options.extend(fields[1:]) + return {"nameservers": nameservers, "search": search, "options": options} + +def default_route_summary(): + content = app_text(["cat", "/proc/net/route"]) + for line in content.splitlines()[1:]: + fields = line.split() + if len(fields) < 4 or fields[1] != "00000000": + continue + try: + gateway = socket.inet_ntoa(struct.pack("= 5 and peer_fields[0] == "ESTAB": + peers.append(peer_fields[4]) + udp_proc = run(["ss", "-unpH"]) + if udp_proc.returncode == 0: + for udp_line in udp_proc.stdout.decode("utf-8", errors="replace").splitlines(): + if f"pid={pid}," not in udp_line: + continue + udp_fields = udp_line.split() + if len(udp_fields) >= 5 and udp_fields[4] not in ("*:*", "0.0.0.0:*"): + udp_peers.append(udp_fields[4]) + cgroup_proc = run(["cat", f"/proc/{pid}/cgroup"]) + cgroup_text = cgroup_proc.stdout.decode("utf-8", errors="replace") if cgroup_proc.returncode == 0 else "" + unit_match = re.search(r'/([^/]+\\.service)(?:/|$)', cgroup_text) + systemd_unit = unit_match.group(1) if unit_match else None + if systemd_unit: + show_proc = run(["systemctl", "show", systemd_unit, "--property=ActiveState", "--property=SubState", "--property=NRestarts"]) + if show_proc.returncode == 0: + service_values = dict(line.split("=", 1) for line in show_proc.stdout.decode("utf-8", errors="replace").splitlines() if "=" in line) + active_state = "/".join(value for value in (service_values.get("ActiveState"), service_values.get("SubState")) if value) + service_restarts = service_values.get("NRestarts") + rows.append({ + "localEndpoint": local_endpoint, + "processName": process_name, + "pid": pid, + "executable": executable, + "establishedPeers": sorted(set(peers))[:12], + "udpPeers": sorted(set(udp_peers))[:12], + "systemdUnit": systemd_unit, + "activeState": active_state, + "serviceRestarts": service_restarts, + }) + return {"status": "available", "listeners": rows, "credentialsPrinted": False} + +def proxy_service_evidence(owner): + listeners = owner.get("listeners") if isinstance(owner, dict) else [] + units = sorted(set(item.get("systemdUnit") for item in listeners if isinstance(item, dict) and item.get("systemdUnit"))) + if not units: + return {"status": "not-applicable", "buckets": [], "samples": []} + buckets = {} + samples = [] + for unit in units: + proc = run(["journalctl", "--unit", unit, "--since", since_start_time(PAYLOAD.get("since")), "--no-pager", "--output", "json", "--lines", "2000"]) + if proc.returncode != 0: + continue + for line in proc.stdout.decode("utf-8", errors="replace").splitlines(): + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + message = str(item.get("MESSAGE") or "") + lowered = message.lower() + priority = int(item.get("PRIORITY")) if str(item.get("PRIORITY") or "").isdigit() else 7 + cause = None + if "reconnect" in lowered or "retry" in lowered: cause = "reconnect" + elif "disconnect" in lowered or "closed" in lowered: cause = "disconnected" + elif "timeout" in lowered or "deadline" in lowered: cause = "timeout" + elif "error" in lowered or "failed" in lowered or priority <= 3: cause = "service-error" + if cause is None: + continue + buckets[cause] = buckets.get(cause, 0) + 1 + if len(samples) < (30 if PAYLOAD.get("full") is True else 12): + samples.append({"at": item.get("__REALTIME_TIMESTAMP"), "cause": cause, "message": text(message, 400)}) + return { + "status": "available", + "units": units, + "buckets": [{"cause": key, "count": value} for key, value in sorted(buckets.items(), key=lambda pair: (-pair[1], pair[0]))], + "samples": samples, + } + +def classify_network_message(message, status_code): + lowered = message.lower() + checks = ( + ("dns-failure", ("no such host", "name resolution", "temporary failure in name resolution")), + ("proxy-connect-failure", ("proxyconnect", "connect tunnel", "http connect")), + ("connection-reset", ("connection reset", "reset by peer")), + ("unexpected-eof", ("unexpected eof", " eof")), + ("broken-pipe", ("broken pipe",)), + ("connection-refused", ("connection refused",)), + ("network-unreachable", ("network is unreachable", "no route to host")), + ("transport-timeout", ("i/o timeout", "tls handshake timeout", "context deadline exceeded")), + ("context-canceled", ("context canceled", "context cancelled")), + ("stream-incomplete", ("missing terminal event", "stream usage incomplete", "response failed")), + ) + for cause, markers in checks: + if any(marker in lowered for marker in markers): + return cause + return "upstream-503" if status_code == 503 or " 503" in lowered else None + +def recent_network_evidence(account_id): + lines = runtime_log_lines() + buckets = {} + samples = [] + matched = 0 + sample_limit = 30 if PAYLOAD.get("full") is True else 12 + for raw in lines: + candidate = raw[raw.find("{"):] if "{" in raw else raw + try: + item = json.loads(candidate) + except json.JSONDecodeError: + item = {} + extra = item.get("extra") if isinstance(item.get("extra"), dict) else {} + observed_account = item.get("account_id") if item.get("account_id") is not None else extra.get("account_id") + if str(observed_account) != str(account_id): + continue + status_code = item.get("status_code") if isinstance(item.get("status_code"), int) else extra.get("status_code") + if not isinstance(status_code, int): + upstream_status = extra.get("upstream_status") + status_code = upstream_status if isinstance(upstream_status, int) else None + message_parts = [item.get("message"), item.get("error"), extra.get("error"), extra.get("message")] + message = " | ".join(str(value) for value in message_parts if value not in (None, "")) + cause = classify_network_message(message, status_code) + if cause is None: + continue + matched += 1 + buckets[cause] = buckets.get(cause, 0) + 1 + if len(samples) < sample_limit: + samples.append({ + "at": item.get("time") or item.get("created_at") or extra.get("_at"), + "requestId": item.get("request_id") or extra.get("request_id"), + "cause": cause, + "statusCode": status_code, + "message": text(message, 360), + }) + return { + "source": "sub2api-target-runtime-logs", + "window": PAYLOAD.get("since"), + "scannedLines": len(lines), + "matchedEvents": matched, + "buckets": [{"cause": key, "count": value} for key, value in sorted(buckets.items(), key=lambda pair: (-pair[1], pair[0]))], + "samples": samples, + } + +def infrastructure_snapshot(token, detail): + proxy = proxy_summary(token, detail.get("proxy_id")) + proxy_env, no_proxy = proxy_environment_summary() + owner = proxy_endpoint_owner(proxy) + return { + "source": "sub2api-admin-api-and-target-runtime", + "accountProxy": proxy, + "appRuntime": app_runtime_summary(), + "proxyEnvironment": proxy_env, + "noProxy": no_proxy, + "dns": dns_summary(), + "defaultRoute": default_route_summary(), + "interfaces": interface_summaries(), + "proxySockets": proxy_socket_summary(proxy), + "proxyEndpointOwner": owner, + "proxyServiceEvidence": proxy_service_evidence(owner), + "recentNetworkEvidence": recent_network_evidence(detail.get("id")), + "attribution": "The account proxy record controls this account's transport. Container proxy environment is reported separately. Interface counters and sockets are query-time snapshots; no upstream account test or application request is sent.", + "valuesPrinted": False, + } + +def policy_summary(policy, include_rules=False): + keys = ("enabled", "ruleCount", "statusCodes", "rules") if include_rules else ("enabled", "ruleCount", "statusCodes") + return {key: policy[key] for key in keys} + +def find_account(token, accounts, selector): + exact = [item for item in accounts if str(item.get("id")) == selector or item.get("name") == selector] + if len(exact) != 1: + raise RuntimeError("account-not-found-or-ambiguous: " + selector) + return account_detail(token, exact[0]["id"]) + +def runtime_log_lines(): + since = PAYLOAD.get("since", "24h") + tail = str(PAYLOAD.get("tail", 50000)) + if RUNTIME_MODE == "host-docker": + proc = docker(["logs", "--since", since, "--tail", tail, APP_POD]) + else: + proc = kubectl(["-n", NAMESPACE, "logs", APP_POD, "--since=" + since, "--tail=" + tail]) + if proc.returncode != 0: + raise RuntimeError("read Sub2API runtime logs failed: " + text(proc.stderr)) + output = proc.stdout + b"\\n" + proc.stderr + return output.decode("utf-8", errors="replace").splitlines() + +def app_exec(args): + if RUNTIME_MODE == "host-docker": + return docker(["exec", APP_POD, *args]) + return kubectl(["-n", NAMESPACE, "exec", APP_POD, "--", *args]) + +def app_text(args): + proc = app_exec(args) + return proc.stdout.decode("utf-8", errors="replace").strip() if proc.returncode == 0 else "" + +def config_scalar(path, key): + script = r'''awk -v wanted="$2" ' +BEGIN { in_gateway=0 } +/^gateway:[[:space:]]*($|#)/ { in_gateway=1; next } +in_gateway && /^[^[:space:]#]/ { exit } +in_gateway && $0 ~ "^[[:space:]]+" wanted ":[[:space:]]*" { + value=$0 + sub("^[[:space:]]+" wanted ":[[:space:]]*", "", value) + sub("[[:space:]]*#.*$", "", value) + gsub("^[[:space:]\\\"'\''']+|[[:space:]\\\"'\''']+$", "", value) + print value + exit +}' "$1"''' + return app_text(["sh", "-c", script, "sh", path, key]) + +def runtime_response_header_timeout(): + env_openai = app_text(["sh", "-c", "printenv GATEWAY_OPENAI_RESPONSE_HEADER_TIMEOUT 2>/dev/null || true"]) + env_generic = app_text(["sh", "-c", "printenv GATEWAY_RESPONSE_HEADER_TIMEOUT 2>/dev/null || true"]) + config_file = app_text(["sh", "-c", "printenv CONFIG_FILE 2>/dev/null || true"]) + paths = [value for value in (config_file, "/app/data/config.yaml", "/etc/sub2api/config.yaml", "/app/config.yaml") if value] + config_openai = "" + config_generic = "" + config_source = None + for path in dict.fromkeys(paths): + openai_value = config_scalar(path, "openai_response_header_timeout") + generic_value = config_scalar(path, "response_header_timeout") + if openai_value or generic_value: + config_openai = openai_value + config_generic = generic_value + config_source = path + break + def non_negative_int(value): + try: + parsed = int(str(value).strip()) + return parsed if parsed >= 0 else None + except (TypeError, ValueError): + return None + openai_configured = non_negative_int(config_openai) + openai_env = non_negative_int(env_openai) + generic_configured = non_negative_int(config_generic) + generic_env = non_negative_int(env_generic) + effective = openai_configured if openai_configured is not None else openai_env if openai_env is not None else 0 + source = "config-file:" + config_source if openai_configured is not None else "process-env" if openai_env is not None else "sub2api-default" + generic = generic_configured if generic_configured is not None else generic_env if generic_env is not None else 600 + return { + "effectiveSeconds": effective, + "disabled": effective == 0, + "source": source, + "genericResponseHeaderTimeoutSeconds": generic, + "genericAppliesToOpenAI": False, + "configFileChecked": config_source, + "valuesPrinted": False, + } + +def compact_error(value): + if not isinstance(value, str): + return "" + compact = " ".join(value.replace("\x00", "").split()) + compact = re.sub(r"(?i)\\bBearer\\s+[A-Za-z0-9._~+/=-]+", "Bearer [REDACTED]", compact) + compact = re.sub(r"\\bsk-[A-Za-z0-9_-]{8,}\\b", "sk-[REDACTED]", compact) + compact = re.sub(r"(?i)\\b(api[_ -]?key|token)[=:]\\s*[^\\s,;]+", r"\\1=[REDACTED]", compact) + return compact[:240] + +def since_start_time(value): + match = re.fullmatch(r"(\\d+)([mhd])", str(value or "")) + if not match: + raise RuntimeError("unsupported runtime error window: " + str(value)) + amount = int(match.group(1)) + unit = match.group(2) + duration = timedelta(minutes=amount) if unit == "m" else timedelta(hours=amount) if unit == "h" else timedelta(days=amount) + return (datetime.now(timezone.utc) - duration).isoformat().replace("+00:00", "Z") + +def ops_error_cause(value): + lower = value.lower() + markers = { + "concurrency": ("concurrency limit", "concurrent request", "too many concurrent", "maximum concurrency", "max concurrency"), + "overload": ("overloaded", "server busy", "temporarily unavailable", "capacity exhausted"), + "model-route-unavailable": ("no available channel for model", "unsupported model", "model not found", "model_not_found"), + "context-window": ("context window", "context length", "maximum context"), + "continuation-payload": ("encrypted_content", "missing_required_parameter", "missing required parameter"), + "invalid-request": ("invalid_request", "invalid request"), + "rate-limit": ("rate limit", "too many requests"), + "access-denied": ("forbidden", "access denied", "permission denied"), + "not-found": ("not found", "does not exist"), + "incomplete-stream": ("ended before a terminal event", "missing terminal event", "before response.completed"), + "transport": ("do request failed", "unexpected eof", "connection reset", "stream read error", "stream closed", "internal_error; received from peer"), + "upstream-api-key-validation": ("failed to validate api key", "api key validation failed"), + "authentication": ("unauthorized", "invalid token", "token revoked", "invalid_grant"), + "generic-upstream-failure": ("upstream request failed", "upstream gateway error"), + } + for cause, values in markers.items(): + if any(marker in lower for marker in values): + return cause + if re.match(r"recovered upstream error \\d+\\b", lower): + return "status-only-upstream-failure" + return "unknown" + +def ops_error_analysis(token, account, log_statuses): + account_id = account.get("id") + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + path = f"/api/v1/admin/ops/upstream-errors?account_id={account_id}&view=all&page=1&page_size=500&start_time={start_time}" + response = curl_api("GET", path, bearer=token) + try: + data = data_of(response, "list account ops errors") + except Exception as exc: + return { + "source": "ops-upstream-errors", + "status": "unavailable", + "logObservedStatusCodes": log_statuses, + "reason": compact_error(str(exc)), + "assessment": "insufficient-evidence", + } + rows = items_of(data) + total = data.get("total", len(rows)) if isinstance(data, dict) else len(rows) + detail_limit = 24 if PAYLOAD.get("full") is True else 8 + analyzed = [] + for row in rows: + message = compact_error(row.get("message") if isinstance(row, dict) else "") + analyzed.append({"row": row, "evidence": message, "cause": ops_error_cause(message)}) + for entry in analyzed[:detail_limit]: + row = entry["row"] + error_id = row.get("id") if isinstance(row, dict) else None + if not isinstance(error_id, int): + continue + detail_response = curl_api("GET", f"/api/v1/admin/ops/upstream-errors/{error_id}", bearer=token) + try: + detail = data_of(detail_response, "get account 500 ops error") + except Exception: + continue + fields = [] + for key in ("message", "upstream_error_message", "upstream_error_detail", "error_body", "upstream_errors"): + value = detail.get(key) if isinstance(detail, dict) else None + if isinstance(value, str) and value.strip(): + fields.append(value) + evidence = compact_error(" ".join(fields)) + entry["evidence"] = evidence + entry["cause"] = ops_error_cause(evidence) + + causes = {} + statuses = {} + messages = {} + models = {} + hours = {} + stream_modes = {"stream": 0, "synchronous": 0, "unknown": 0} + concurrency_count = 0 + for entry in analyzed: + row = entry["row"] + error_id = row.get("id") if isinstance(row, dict) else None + evidence = entry["evidence"] + cause = entry["cause"] + causes[cause] = causes.get(cause, 0) + 1 + list_message = compact_error(row.get("message") if isinstance(row, dict) else "") + if list_message: + messages[list_message] = messages.get(list_message, 0) + 1 + status_code = row.get("status_code") if isinstance(row, dict) else None + if isinstance(status_code, int): + statuses[str(status_code)] = statuses.get(str(status_code), 0) + 1 + if cause == "concurrency": + concurrency_count += 1 + model = str(row.get("model") or "unknown") if isinstance(row, dict) else "unknown" + models[model] = models.get(model, 0) + 1 + created_at = str(row.get("created_at") or "") if isinstance(row, dict) else "" + hour = created_at[:13] + ":00" if len(created_at) >= 13 else "unknown" + hours[hour] = hours.get(hour, 0) + 1 + stream = row.get("stream") if isinstance(row, dict) else None + stream_key = "stream" if stream is True else "synchronous" if stream is False else "unknown" + stream_modes[stream_key] += 1 + sample = { + "errorId": error_id, + "createdAt": row.get("created_at") if isinstance(row, dict) else None, + "requestId": row.get("request_id") if isinstance(row, dict) else None, + "model": row.get("model") if isinstance(row, dict) else None, + "stream": row.get("stream") if isinstance(row, dict) else None, + "cause": cause, + "evidence": evidence, + } + entry["sample"] = sample + analyzed_count = len(analyzed) + if analyzed_count == 0: + assessment = "insufficient-evidence" + elif concurrency_count == analyzed_count: + assessment = "concurrency-related" + elif concurrency_count > 0: + assessment = "mixed" + else: + assessment = "no-direct-concurrency-evidence" + dominant_cause = None + dominant_count = 0 + if causes: + dominant_cause, dominant_count = max(causes.items(), key=lambda pair: pair[1]) + dominant_share = round(dominant_count / analyzed_count, 4) if analyzed_count > 0 else None + if dominant_cause == "upstream-api-key-validation": + finding = "The upstream gateway returned HTTP 500 while validating its inbound API key; investigate its key lookup and storage dependencies first." + elif dominant_cause == "concurrency": + finding = "Direct concurrency-limit messages dominate the observed Ops records." + elif dominant_cause is not None: + finding = "The dominant observed Ops cause is " + dominant_cause + "." + else: + finding = "No structured HTTP 500 Ops records were available for cause analysis." + if concurrency_count > 0: + concurrency_action = "consider-lower-limit-after-controlled-comparison" + else: + concurrency_action = "do-not-lower-limit-as-primary-fix" + ranked_messages = sorted(messages.items(), key=lambda pair: (-pair[1], pair[0])) + message_limit = 20 if PAYLOAD.get("full") is True else 5 + return { + "source": "ops-upstream-errors", + "status": "available", + "logObservedStatusCodes": log_statuses, + "opsRecordCount": total, + "analyzedRecordCount": analyzed_count, + "detailRecordCount": min(detail_limit, analyzed_count), + "recordsTruncated": total > analyzed_count, + "statusBuckets": [{"statusCode": int(code), "count": count} for code, count in sorted(statuses.items(), key=lambda pair: int(pair[0]))], + "causeBuckets": [{"cause": cause, "count": count} for cause, count in sorted(causes.items())], + "messageBuckets": [{"message": message, "count": count} for message, count in ranked_messages[:message_limit]], + "concurrencyRelatedRecordCount": concurrency_count, + "assessment": assessment, + "rootCause": { + "dominantCause": dominant_cause, + "dominantCount": dominant_count, + "dominantShare": dominant_share, + "finding": finding, + }, + "errorRateEvidence": { + "errorRecordCount": total, + "requestAttemptDenominator": None, + "errorRate": None, + "reason": "The available Ops endpoint lists errors but does not provide all routing attempts for this account.", + }, + "concurrencyRecommendation": { + "action": concurrency_action, + "reason": "No error-time concurrency history is available; only explicit concurrency error text is treated as direct evidence.", + }, + "runtimeConcurrency": { + "limit": account.get("concurrency"), + "current": account.get("current_concurrency"), + "loadFactor": account.get("load_factor"), + "historicalAtErrorAvailable": False, + }, + **({ + "modelBuckets": [{"model": model, "count": count} for model, count in sorted(models.items(), key=lambda pair: (-pair[1], pair[0]))], + "hourBuckets": [{"hour": hour, "count": count} for hour, count in sorted(hours.items())], + "streamBuckets": [{"mode": mode, "count": count} for mode, count in stream_modes.items() if count > 0], + "samples": [entry["sample"] for entry in analyzed[:detail_limit]], + } if PAYLOAD.get("full") is True else { + "samples": [entry["sample"] for entry in analyzed[:1]], + }), + "disclosure": "Ops records are independently sampled evidence and are not positionally joined to account_upstream_error log lines.", + } + +def gateway_request_path(value): + path = str(value or "").split("?", 1)[0].rstrip("/") + return path in ( + "/responses", "/v1/responses", "/responses/compact", "/v1/responses/compact", + "/messages", "/v1/messages", "/chat/completions", "/v1/chat/completions", + ) + +def native_ops_snapshot(token, group_id, platform=None): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + base_query = f"group_id={group_id}&start_time={start_time}{platform_query}" + + def optional(path, label): + response = curl_api("GET", path, bearer=token) + try: + return {"status": "available", "data": data_of(response, label)} + except Exception as exc: + return {"status": "unavailable", "reason": compact_error(str(exc))} + + overview_result = optional(f"/api/v1/admin/ops/dashboard/overview?{base_query}&query_mode=raw", "get ops dashboard overview") + availability_result = optional(f"/api/v1/admin/ops/account-availability?group_id={group_id}{platform_query}", "get ops account availability") + concurrency_result = optional(f"/api/v1/admin/ops/concurrency?group_id={group_id}{platform_query}", "get ops concurrency") + sink_result = optional("/api/v1/admin/ops/system-logs/health", "get ops system log health") + + overview = overview_result.get("data") if overview_result.get("status") == "available" else None + overview_summary = None + if isinstance(overview, dict): + overview_summary = { + key: overview.get(key) + for key in ( + "start_time", "end_time", "health_score", "success_count", "error_count_total", + "business_limited_count", "error_count_sla", "request_count_total", "request_count_sla", + "error_rate", "upstream_error_rate", "upstream_error_count_excl_429_529", + "upstream_429_count", "upstream_529_count", + "duration", "ttft", + ) + } + + availability = availability_result.get("data") if availability_result.get("status") == "available" else None + availability_summary = None + if isinstance(availability, dict): + group_values = availability.get("group") if isinstance(availability.get("group"), dict) else {} + account_values = availability.get("account") if isinstance(availability.get("account"), dict) else {} + selected_group = group_values.get(str(group_id)) or group_values.get(group_id) + unavailable = [] + for value in account_values.values(): + if not isinstance(value, dict) or value.get("group_id") != group_id or value.get("is_available") is True: + continue + unavailable.append({ + "accountId": value.get("account_id"), + "accountName": value.get("account_name"), + "status": value.get("status"), + "rateLimited": value.get("is_rate_limited"), + "rateLimitRemainingSeconds": value.get("rate_limit_remaining_sec"), + "overloaded": value.get("is_overloaded"), + "overloadRemainingSeconds": value.get("overload_remaining_sec"), + "hasError": value.get("has_error"), + "error": compact_error(value.get("error_message")), + }) + availability_summary = { + "enabled": availability.get("enabled"), + "timestamp": availability.get("timestamp"), + "group": selected_group, + "unavailableAccounts": unavailable[:20 if PAYLOAD.get("full") is True else 8], + "unavailableAccountCount": max( + len(unavailable), + max(0, selected_group.get("total_accounts") - selected_group.get("available_count")) + if isinstance(selected_group, dict) + and isinstance(selected_group.get("total_accounts"), int) + and isinstance(selected_group.get("available_count"), int) + else 0, + ), + "unavailableAccountDetailsComplete": ( + not isinstance(selected_group, dict) + or not isinstance(selected_group.get("total_accounts"), int) + or not isinstance(selected_group.get("available_count"), int) + or len(unavailable) >= max(0, selected_group.get("total_accounts") - selected_group.get("available_count")) + ), + } + + concurrency = concurrency_result.get("data") if concurrency_result.get("status") == "available" else None + concurrency_summary = None + if isinstance(concurrency, dict): + group_values = concurrency.get("group") if isinstance(concurrency.get("group"), dict) else {} + account_values = concurrency.get("account") if isinstance(concurrency.get("account"), dict) else {} + selected_group = group_values.get(str(group_id)) or group_values.get(group_id) + loaded = [] + for value in account_values.values(): + if not isinstance(value, dict) or value.get("group_id") != group_id: + continue + if (value.get("current_in_use") or 0) <= 0 and (value.get("waiting_in_queue") or 0) <= 0: + continue + loaded.append({ + "accountId": value.get("account_id"), + "accountName": value.get("account_name"), + "currentInUse": value.get("current_in_use"), + "maxCapacity": value.get("max_capacity"), + "loadPercentage": value.get("load_percentage"), + "waitingInQueue": value.get("waiting_in_queue"), + }) + concurrency_summary = { + "enabled": concurrency.get("enabled"), + "timestamp": concurrency.get("timestamp"), + "group": selected_group, + "activeAccounts": sorted(loaded, key=lambda item: (-(item.get("loadPercentage") or 0), str(item.get("accountName") or "")))[:20 if PAYLOAD.get("full") is True else 8], + } + + sink = sink_result.get("data") if sink_result.get("status") == "available" else None + return { + "source": "sub2api-native-admin-ops", + "overview": {"status": overview_result.get("status"), "summary": overview_summary, "reason": overview_result.get("reason")}, + "accountAvailability": {"status": availability_result.get("status"), "summary": availability_summary, "reason": availability_result.get("reason")}, + "concurrency": {"status": concurrency_result.get("status"), "summary": concurrency_summary, "reason": concurrency_result.get("reason")}, + "systemLogSink": {"status": sink_result.get("status"), "health": sink if isinstance(sink, dict) else None, "reason": sink_result.get("reason")}, + } + +def native_request_timing(token, group_id, platform, overview): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + base = f"group_id={group_id}&start_time={start_time}{platform_query}" + all_counts = {} + error_counts = {} + status = "available" + reason = None + duration = overview.get("duration") if isinstance(overview.get("duration"), dict) else {} + ttft = overview.get("ttft") if isinstance(overview.get("ttft"), dict) else {} + ttft_p99 = ttft.get("p99_ms") + observed_floor = max(60, ((ttft_p99 + 29999) // 15000) * 15) if isinstance(ttft_p99, int) else None + candidate_seconds = sorted(set([30, 45, 60] + ([observed_floor] if isinstance(observed_floor, int) else []))) + error_duration_response = curl_api("GET", f"/api/v1/admin/ops/requests?{base}&page=1&page_size=1&kind=error&min_duration_ms=0", bearer=token) + try: + error_duration_data = data_of(error_duration_response, "get error duration coverage") + error_duration_record_count = error_duration_data.get("total") if isinstance(error_duration_data, dict) else None + except Exception as exc: + error_duration_record_count = None + status = "unavailable" + reason = compact_error(str(exc)) + for seconds in candidate_seconds: + threshold = seconds * 1000 + for kind, target in ((None, all_counts), ("error", error_counts)): + kind_query = "&kind=error" if kind else "" + response = curl_api("GET", f"/api/v1/admin/ops/requests?{base}&page=1&page_size=1&min_duration_ms={threshold}{kind_query}", bearer=token) + try: + data = data_of(response, "get long request count") + value = data.get("total") if isinstance(data, dict) else None + target[threshold] = None if kind == "error" and error_duration_record_count == 0 else value + except Exception as exc: + status = "unavailable" + reason = compact_error(str(exc)) + target[threshold] = None + candidates = [] + for seconds in candidate_seconds: + headroom = seconds * 1000 - ttft_p99 if isinstance(ttft_p99, int) else None + candidates.append({ + "seconds": seconds, + "observedFloor": seconds == observed_floor, + "ttftP99HeadroomMs": headroom, + "ttftP99Below": headroom >= 0 if isinstance(headroom, int) else None, + "totalDurationAtLeastCount": all_counts.get(seconds * 1000), + "errorDurationAtLeastCount": error_counts.get(seconds * 1000), + "assessment": "observed-ttft-p99-plus-15s-rounded" if seconds == observed_floor else "ttft-p99-within-candidate" if isinstance(headroom, int) and headroom >= 0 else "ttft-p99-exceeds-candidate" if isinstance(headroom, int) else "insufficient-ttft-evidence", + }) + since = str(PAYLOAD.get("since") or "") + token_range = {"30m": "30m", "60m": "1h", "1h": "1h", "24h": "1d", "1d": "1d"}.get(since) + model_timing = [] + model_timing_status = "unsupported-window" + if token_range: + response = curl_api("GET", f"/api/v1/admin/ops/dashboard/openai-token-stats?group_id={group_id}&time_range={token_range}&top_n=100{platform_query}", bearer=token) + try: + data = data_of(response, "get OpenAI model token stats") + rows = items_of(data) + for row in rows: + if not isinstance(row, dict): + continue + request_count = row.get("request_count") + first_count = row.get("requests_with_first_token") + model_timing.append({ + "model": row.get("model"), "requestCount": request_count, + "requestsWithFirstToken": first_count, + "firstTokenCoverage": round(first_count / request_count, 6) if isinstance(first_count, int) and isinstance(request_count, int) and request_count > 0 else None, + "avgFirstTokenMs": row.get("avg_first_token_ms"), "avgDurationMs": row.get("avg_duration_ms"), + "avgTokensPerSec": row.get("avg_tokens_per_sec"), + }) + model_timing.sort(key=lambda item: (-(item.get("avgFirstTokenMs") or -1), str(item.get("model") or ""))) + model_timing_status = "available" + except Exception as exc: + model_timing_status = "unavailable" + reason = compact_error(str(exc)) + return { + "status": status, + "reason": reason, + "source": "sub2api-native-admin-ops-dashboard-and-requests", + "ttft": ttft, + "duration": duration, + "observedFloorSeconds": observed_floor, + "errorDurationRecordCount": error_duration_record_count, + "modelTimingStatus": model_timing_status, + "modelTimingWindow": token_range, + "modelTiming": model_timing[:20 if PAYLOAD.get("full") is True else 8], + "candidates": candidates, + "attribution": "TTFT covers successful streaming requests with first_token_ms. Total and error duration are end-to-end request durations, not response-header wait or guaranteed failover lead time. The observed floor is TTFT P99 plus 15 seconds rounded up to 15 seconds; it is not a deployment recommendation. A dash in ERROR_DUR>= means error duration coverage is unavailable.", + } + +`; +} diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-tail.ts b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-tail.ts new file mode 100644 index 00000000..7f7b6674 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script-tail.ts @@ -0,0 +1,1229 @@ +export function runtimeRemoteScriptTail(): string { + return `def native_customer_error_profile(token, group_id, platform): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + page = 1 + page_size = 500 + rows = [] + total = None + try: + while True: + response = curl_api("GET", f"/api/v1/admin/ops/request-errors?group_id={group_id}&start_time={start_time}&view=all&page={page}&page_size={page_size}{platform_query}", bearer=token) + data = data_of(response, "get customer-visible request errors") + page_rows = items_of(data) + rows.extend(page_rows) + if total is None and isinstance(data, dict) and isinstance(data.get("total"), int): + total = data.get("total") + if not page_rows or len(page_rows) < page_size or (isinstance(total, int) and len(rows) >= total): + break + page += 1 + except Exception as exc: + return {"status": "unavailable", "source": "sub2api-native-admin-ops-request-errors", "reason": compact_error(str(exc))} + if total is None: + total = len(rows) + def buckets(key_fn): + counts = {} + for row in rows: + value = key_fn(row) if isinstance(row, dict) else None + label = str(value) if value not in (None, "") else "unknown" + counts[label] = counts.get(label, 0) + 1 + return [{"value": key, "count": count} for key, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20 if PAYLOAD.get("full") is True else 8]] + samples = [] + account_metrics = {} + def customer_error_attribution(row): + message = str(row.get("message") or row.get("error_message") or "").lower() + if "context window" in message or "context_length_exceeded" in message: + return False, "context-window" + if "input must be a list" in message: + return False, "invalid-client-input" + if "not supported by any configured account" in message or "no available channel for model" in message: + return False, "model-route" + phase = str(row.get("phase") or "").lower() + if phase in ("internal", "client", "business"): + return False, "non-upstream-phase" + if row.get("account_id") is None: + return False, "no-account-attribution" + category = " ".join(str(row.get(key) or "").lower() for key in ("type", "category", "error_type")) + if phase == "upstream" or "upstream" in category: + return True, "explicit-upstream" + upstream_messages = ( + "upstream service temporarily unavailable", "upstream request failed", + "bad gateway", "gateway timeout", "error code: 502", "error code: 503", + "error code: 504", "error code: 524", + ) + if any(marker in message for marker in upstream_messages): + return True, "stable-upstream-message" + return False, "unattributed-customer-error" + seen = set() + for row in rows: + if not isinstance(row, dict): + continue + account_id = row.get("account_id") + account_key = str(account_id) if account_id is not None else "unknown" + if account_key not in account_metrics: + account_metrics[account_key] = { + "accountId": account_id, "customerErrorEvents": 0, "customerErrorRequestIds": set(), + "upstreamRequestIds": set(), "excludedRequestIds": set(), "excludedReasons": {}, "statusCodes": {}, + } + metric = account_metrics[account_key] + metric["customerErrorEvents"] += 1 + request_id = row.get("request_id") + if isinstance(request_id, str) and request_id: + metric["customerErrorRequestIds"].add(request_id) + scoreable, reason = customer_error_attribution(row) + target = metric["upstreamRequestIds"] if scoreable else metric["excludedRequestIds"] + target.add(request_id) + if not scoreable: + metric["excludedReasons"][reason] = metric["excludedReasons"].get(reason, 0) + 1 + status_code = row.get("status_code") + if isinstance(status_code, int): + status_key = str(status_code) + metric["statusCodes"][status_key] = metric["statusCodes"].get(status_key, 0) + 1 + if not request_id: + continue + key = (row.get("requested_model") or row.get("model"), row.get("status_code"), row.get("account_id"), row.get("stream"), row.get("request_path") or row.get("inbound_endpoint")) + sample_limit = 30 if PAYLOAD.get("full") is True else 15 + if key in seen or len(samples) >= sample_limit: + continue + seen.add(key) + samples.append({ + "requestId": row.get("request_id"), "model": key[0], "statusCode": key[1], "accountId": key[2], + "mode": "stream" if key[3] is True else "sync" if key[3] is False else "unknown", "path": key[4], + "message": compact_error(row.get("message") or row.get("error_message")), + }) + return { + "status": "available", "source": "sub2api-native-admin-ops-request-errors", "total": total, + "returned": len(rows), "pagesRead": page, "recordsTruncated": False, + "modelBuckets": buckets(lambda row: row.get("requested_model") or row.get("model")), + "accountBuckets": buckets(lambda row: row.get("account_id")), + "statusBuckets": buckets(lambda row: row.get("status_code")), + "streamBuckets": buckets(lambda row: "stream" if row.get("stream") is True else "sync" if row.get("stream") is False else "unknown"), + "messageBuckets": buckets(lambda row: compact_error(row.get("message") or row.get("error_message"))), + "phaseBuckets": buckets(lambda row: row.get("phase")), + "endpointBuckets": buckets(lambda row: row.get("inbound_endpoint") or row.get("request_path")), + "pathBuckets": buckets(lambda row: row.get("request_path") or row.get("inbound_endpoint")), + "accountMetrics": [ + { + "accountId": item["accountId"], "customerErrorEvents": item["customerErrorEvents"], + "customerErrorRequests": len(item["customerErrorRequestIds"]), + "_requestIds": sorted(item["customerErrorRequestIds"]), + "_upstreamRequestIds": sorted(item["upstreamRequestIds"]), + "_excludedRequestIds": sorted(item["excludedRequestIds"]), + "scoreableUpstreamErrorRequests": len(item["upstreamRequestIds"]), + "excludedNonUpstreamErrorRequests": len(item["excludedRequestIds"]), + "excludedReasonBuckets": [{"reason": reason, "count": count} for reason, count in sorted(item["excludedReasons"].items())], + "statusBuckets": [ + {"statusCode": int(code), "count": count} + for code, count in sorted(item["statusCodes"].items(), key=lambda pair: int(pair[0])) + ], + } + for item in sorted(account_metrics.values(), key=lambda item: int(item.get("accountId") or 0)) + ], + "samples": samples, + } + +def parse_native_time(value): + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc) + except ValueError: + return None + +def metric_percentile(values, fraction): + ordered = sorted(value for value in values if isinstance(value, (int, float)) and value >= 0) + if not ordered: + return None + position = (len(ordered) - 1) * fraction + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + return round(ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)) + +def numeric_value(value): + try: + return float(value or 0) + except (TypeError, ValueError): + return 0.0 + +def account_cost_rate(account_name): + match = re.search(r"(\\d+(?:\\.\\d+)?)$", str(account_name or "")) + return float(match.group(1)) if match else None + +def account_usage_metrics(token, accounts, group_id): + start_at = parse_native_time(since_start_time(PAYLOAD.get("since"))) + end_at = datetime.now(timezone.utc) + result = {} + for account in accounts: + account_id = account.get("id") + key = str(account_id) + rows = [] + page = 1 + status = "available" + reason = None + try: + while True: + path = ( + f"/api/v1/admin/usage?page={page}&page_size=100&group_id={group_id}&account_id={account_id}" + f"&start_date={start_at.date().isoformat()}&end_date={end_at.date().isoformat()}&timezone=UTC" + "&sort_by=created_at&sort_order=desc" + ) + data = data_of(curl_api("GET", path, bearer=token), "list account usage") + page_rows = items_of(data) + reached_start = False + for row in page_rows: + created_at = parse_native_time(row.get("created_at") if isinstance(row, dict) else None) + if created_at is None or created_at < start_at or created_at > end_at: + if created_at is not None and created_at < start_at: + reached_start = True + continue + rows.append(row) + if not page_rows or len(page_rows) < 100 or reached_start: + break + page += 1 + except Exception as exc: + status = "unavailable" + reason = compact_error(str(exc)) + stream_rows = [row for row in rows if isinstance(row, dict) and row.get("stream") is True] + ttft_values = [row.get("first_token_ms") for row in stream_rows if isinstance(row.get("first_token_ms"), int)] + duration_values = [row.get("duration_ms") for row in rows if isinstance(row.get("duration_ms"), int)] + models = {} + for row in rows: + model = str(row.get("model") or "unknown") + models[model] = models.get(model, 0) + 1 + result[key] = { + "status": status, "reason": reason, "source": "sub2api-native-admin-usage", + "pagesRead": page, "successRequests": len(rows), "streamSuccessRequests": len(stream_rows), + "requestCount": len(rows), + "tokenCount": sum(int(row.get("input_tokens") or 0) + int(row.get("output_tokens") or 0) for row in rows), + "apiAmountUsd": round(sum(numeric_value(row.get("actual_cost")) for row in rows), 8), + "firstTokenSamples": len(ttft_values), + "firstTokenCoverage": round(len(ttft_values) / len(stream_rows), 6) if stream_rows else None, + "ttftP50Ms": metric_percentile(ttft_values, 0.50), "ttftP95Ms": metric_percentile(ttft_values, 0.95), + "ttftP99Ms": metric_percentile(ttft_values, 0.99), "ttftMaxMs": max(ttft_values) if ttft_values else None, + "durationP95Ms": metric_percentile(duration_values, 0.95), + "modelBuckets": [{"model": model, "count": count} for model, count in sorted(models.items(), key=lambda item: (-item[1], item[0]))[:8]], + } + return result + +def reliability_score_points(failure_rate): + if not isinstance(failure_rate, (int, float)): + return None + return round(60 * (1 - min(max(failure_rate, 0), 0.20) / 0.20), 2) + +def latency_score_points(ttft_p95_ms): + if not isinstance(ttft_p95_ms, (int, float)): + return None + return round(25 * (1 - min(max(ttft_p95_ms - 10000, 0), 170000) / 170000), 2) + +def account_quality_scores(token, accounts, group_id, native_ops, effectiveness): + usage_by_account = account_usage_metrics(token, accounts, group_id) + customer_profile = native_ops.get("customerErrors") if isinstance(native_ops.get("customerErrors"), dict) else {} + customer_by_account = {str(item.get("accountId")): item for item in customer_profile.get("accountMetrics") or [] if isinstance(item, dict)} + policy_by_account = {str(item.get("accountId")): item for item in effectiveness.get("accountPolicyMetrics") or [] if isinstance(item, dict)} + availability = ((native_ops.get("accountAvailability") or {}).get("summary") or {}) + unavailable_ids = set(str(item.get("accountId")) for item in availability.get("unavailableAccounts") or [] if isinstance(item, dict)) + rows = [] + for account in accounts: + account_id = account.get("id") + key = str(account_id) + usage = usage_by_account.get(key) or {} + customer = customer_by_account.get(key) or {} + policy = policy_by_account.get(key) or {} + policy_failure_ids = set(policy.pop("_failureRequestIds", []) or []) + customer_failure_ids = set(customer.get("_upstreamRequestIds", []) or []) + excluded_failure_ids = set(customer.get("_excludedRequestIds", []) or []) + failure_requests = len((policy_failure_ids | customer_failure_ids) - excluded_failure_ids) + success_requests = usage.get("successRequests") if isinstance(usage.get("successRequests"), int) else 0 + observed_attempts = success_requests + failure_requests + failure_rate = round(failure_requests / observed_attempts, 6) if observed_attempts > 0 else None + reliability_points = reliability_score_points(failure_rate) + ttft_samples = usage.get("firstTokenSamples") if isinstance(usage.get("firstTokenSamples"), int) else 0 + latency_points = latency_score_points(usage.get("ttftP95Ms")) if ttft_samples >= 5 else None + current_available = key not in unavailable_ids and account.get("status") == "active" and account.get("schedulable") is not False + availability_points = 15 if current_available else 8 if account.get("status") == "active" else 0 + earned = (reliability_points or 0) + (latency_points or 0) + availability_points + available_weight = (60 if reliability_points is not None else 0) + (25 if latency_points is not None else 0) + 15 + score = round(earned / available_weight * 100, 1) if observed_attempts > 0 and available_weight > 0 else None + score_comparable = observed_attempts >= 10 and ttft_samples >= 5 + grade = "insufficient" + if isinstance(score, (int, float)) and (score_comparable or (score < 60 and observed_attempts >= 10)): + grade = "A" if score >= 90 else "B" if score >= 80 else "C" if score >= 70 else "D" if score >= 60 else "E" + assessment = "preferred" if grade == "A" else "healthy" if grade == "B" else "watch" if grade == "C" else "degraded" if grade == "D" else "poor" if grade == "E" else "insufficient-evidence" + confidence = "high" if observed_attempts >= 50 and ttft_samples >= 20 else "medium" if observed_attempts >= 10 and ttft_samples >= 5 else "low" + reasons = [] + if isinstance(failure_rate, (int, float)) and failure_rate >= 0.10: + reasons.append("failure-rate>=10%") + elif isinstance(failure_rate, (int, float)) and failure_rate >= 0.03: + reasons.append("failure-rate>=3%") + if isinstance(usage.get("ttftP95Ms"), int) and usage.get("ttftP95Ms") >= 120000: + reasons.append("ttft-p95>=120s") + if policy.get("failoverRequests", 0) > 0: + reasons.append("upstream-failover-triggered") + if policy.get("sameAccountRetryEvents", 0) > 0: + reasons.append("same-account-retry-observed") + if not current_available: + reasons.append("currently-unavailable") + if ttft_samples < 5: + reasons.append("ttft-evidence-insufficient") + if observed_attempts < 10: + reasons.append("request-evidence-insufficient") + cost_rate = account_cost_rate(account.get("name")) + api_amount = usage.get("apiAmountUsd") + rows.append({ + "accountId": account_id, "accountName": account.get("name"), "status": account.get("status"), + "schedulable": account.get("schedulable"), "currentlyAvailable": current_available, + "score": score, "grade": grade, "assessment": assessment, "confidence": confidence, + "scoreComparable": score_comparable, + "observedAttempts": observed_attempts, "successRequests": success_requests, + "failureRequests": failure_requests, "failureRate": failure_rate, + "streamSuccessRequests": usage.get("streamSuccessRequests"), "firstTokenSamples": ttft_samples, + "firstTokenCoverage": usage.get("firstTokenCoverage"), "ttftP50Ms": usage.get("ttftP50Ms"), + "ttftP95Ms": usage.get("ttftP95Ms"), "ttftP99Ms": usage.get("ttftP99Ms"), "ttftMaxMs": usage.get("ttftMaxMs"), + "durationP95Ms": usage.get("durationP95Ms"), "modelBuckets": usage.get("modelBuckets"), + "customerErrorRequests": customer.get("customerErrorRequests", 0), + "scoreableUpstreamErrorRequests": customer.get("scoreableUpstreamErrorRequests", 0), + "excludedNonUpstreamErrorRequests": customer.get("excludedNonUpstreamErrorRequests", 0), + "excludedReasonBuckets": customer.get("excludedReasonBuckets", []), + "usage": { + "requestCount": usage.get("requestCount"), "tokenCount": usage.get("tokenCount"), + "apiAmountUsd": api_amount, "costRateCnyPerApiUsd": cost_rate, + "upstreamCostCny": round(api_amount * cost_rate, 8) if isinstance(api_amount, (int, float)) and isinstance(cost_rate, (int, float)) else None, + }, + "failoverRequests": policy.get("failoverRequests", 0), "failoverRecovered": policy.get("failoverRecovered", 0), + "failoverFailed": policy.get("failoverFailed", 0), "failoverOutcomeMissing": policy.get("failoverOutcomeMissing", 0), + "sameAccountRetryEvents": policy.get("sameAccountRetryEvents", 0), + "tempUnschedulableEvents": policy.get("tempUnschedulableEvents", 0), + "forwardFailedRequests": policy.get("forwardFailedRequests", 0), + "upstreamStatusBuckets": policy.get("upstreamStatusBuckets", []), + "scoreComponents": {"reliability": reliability_points, "latency": latency_points, "availability": availability_points, "availableWeight": available_weight}, + "reasons": reasons, "usageStatus": usage.get("status"), "usageReason": usage.get("reason"), + }) + rows.sort(key=lambda item: (item.get("score") is None, -(item.get("score") or -1), int(item.get("accountId") or 0))) + return { + "source": "sub2api-native-admin-usage-request-errors-and-system-logs", + "scope": "group-account-window", "window": PAYLOAD.get("since"), + "scoreRule": { + "reliabilityWeight": 60, "latencyWeight": 25, "availabilityWeight": 15, + "failureDefinition": "unique request ids with account failover, forward_failed, or explicitly attributable upstream customer error; client input, model-route, internal/business, no-account, and unattributed errors are excluded", + "reliability": "linear from 60 points at 0% observed failures to 0 points at 20%", + "latency": "requires at least 5 first-token samples; linear from 25 points at <=10s TTFT P95 to 0 points at >=180s", + "availability": "15 points when active, schedulable, and currently available; 8 when active but unavailable; otherwise 0", + "missingMetric": "missing latency is excluded from the denominator and lowers confidence; it is not treated as zero latency", + }, + "accounts": rows, + "attribution": "Success denominators, usage, token count, API amount, and first-token latency come from native admin usage. Reliability only includes explicitly attributable upstream customer errors and deduplicated failover/forward request ids; all excluded reasons remain visible.", + } + +def all_group_error_overview(token, groups): + page_size = 20 + cursor = int(PAYLOAD.get("pageToken") or 0) + ordered = sorted( + [item for item in groups if isinstance(item, dict) and isinstance(item.get("id"), int)], + key=lambda item: item["id"], + ) + remaining = [item for item in ordered if item["id"] > cursor] + page = remaining[:page_size] + summaries = [] + totals = { + "requestCount": 0, + "errorCount": 0, + "businessLimitedCount": 0, + "upstreamErrorCount": 0, + "groupsNeedingAttention": 0, + } + for group in page: + group_id = group["id"] + platform = PAYLOAD.get("platform") or group.get("platform") + snapshot = native_ops_snapshot(token, group_id, platform) + overview = ((snapshot.get("overview") or {}).get("summary") or {}) + availability = ((snapshot.get("accountAvailability") or {}).get("summary") or {}) + availability_group = availability.get("group") if isinstance(availability.get("group"), dict) else {} + concurrency = ((snapshot.get("concurrency") or {}).get("summary") or {}) + concurrency_group = concurrency.get("group") if isinstance(concurrency.get("group"), dict) else {} + request_count = overview.get("request_count_total") + error_count = overview.get("error_count_total") + business_limited = overview.get("business_limited_count") + upstream_error_count = sum( + value for value in ( + overview.get("upstream_error_count_excl_429_529"), + overview.get("upstream_429_count"), + overview.get("upstream_529_count"), + ) if isinstance(value, int) + ) + unavailable_count = availability.get("unavailableAccountCount") + needs_attention = ( + (isinstance(error_count, int) and error_count > 0) + or upstream_error_count > 0 + or (isinstance(unavailable_count, int) and unavailable_count > 0) + ) + totals["requestCount"] += request_count if isinstance(request_count, int) else 0 + totals["errorCount"] += error_count if isinstance(error_count, int) else 0 + totals["businessLimitedCount"] += business_limited if isinstance(business_limited, int) else 0 + totals["upstreamErrorCount"] += upstream_error_count + totals["groupsNeedingAttention"] += 1 if needs_attention else 0 + summaries.append({ + "groupId": group_id, + "groupName": group.get("name"), + "platform": platform, + "status": group.get("status"), + "requestCount": request_count, + "errorCount": error_count, + "errorRate": overview.get("error_rate"), + "upstreamErrorCount": upstream_error_count, + "upstreamErrorRate": overview.get("upstream_error_rate"), + "businessLimitedCount": business_limited, + "ttftP99Ms": ((overview.get("ttft") or {}).get("p99_ms") if isinstance(overview.get("ttft"), dict) else None), + "durationP99Ms": ((overview.get("duration") or {}).get("p99_ms") if isinstance(overview.get("duration"), dict) else None), + "totalAccounts": availability_group.get("total_accounts"), + "availableCount": availability_group.get("available_count"), + "rateLimitCount": availability_group.get("rate_limit_count"), + "errorAccountCount": availability_group.get("error_count"), + "unavailableAccountCount": unavailable_count, + "currentInUse": concurrency_group.get("current_in_use"), + "maxCapacity": concurrency_group.get("max_capacity"), + "waitingInQueue": concurrency_group.get("waiting_in_queue"), + "needsAttention": needs_attention, + "opsStatus": { + "overview": (snapshot.get("overview") or {}).get("status"), + "accountAvailability": (snapshot.get("accountAvailability") or {}).get("status"), + "concurrency": (snapshot.get("concurrency") or {}).get("status"), + }, + }) + next_token = str(page[-1]["id"]) if len(remaining) > page_size and page else None + totals["errorRate"] = round(totals["errorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None + totals["upstreamErrorRate"] = round(totals["upstreamErrorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None + ttft_p99_values = [item.get("ttftP99Ms") for item in summaries if isinstance(item.get("ttftP99Ms"), int)] + max_ttft_p99 = max(ttft_p99_values) if ttft_p99_values else None + observed_floor = max(60, ((max_ttft_p99 + 29999) // 15000) * 15) if isinstance(max_ttft_p99, int) else None + return { + "scope": "all-groups-overview", + "window": {"since": PAYLOAD.get("since")}, + "platform": PAYLOAD.get("platform"), + "totalGroups": len(ordered), + "returnedGroups": len(summaries), + "pageSize": page_size, + "pageToken": str(cursor) if cursor > 0 else None, + "nextPageToken": next_token, + "pageTotals": totals, + "responseHeaderTimeout": runtime_response_header_timeout(), + "requestTiming": { + "maxTtftP99Ms": max_ttft_p99, + "observedFloorSeconds": observed_floor, + "scope": "returned-groups-page", + "attribution": "The observed floor is the maximum returned-group TTFT P99 plus 15 seconds rounded up to 15 seconds. It is analytical only and must not be treated as a safe timeout.", + }, + "groups": summaries, + "disclosure": "Page totals cover only returned groups. Follow nextPageToken for remaining groups and use --group for account root causes and request indexes.", + } + +def native_system_log_lines(token, platform=None): + start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") + platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" + markers = ( + "account_temp_unschedulable", "openai.upstream_failover_switching", + "openai.account_select_failed", "openai.forward_failed", + "openai.pool_mode_same_account_retry", "gateway.failover_same_account_retry", + "http request completed", + ) + rows_by_id = {} + queries = [] + for marker in markers: + page = 1 + returned = 0 + total = None + try: + while True: + path = f"/api/v1/admin/ops/system-logs?page={page}&page_size=200&start_time={start_time}&q={quote(marker, safe='')}{platform_query}" + data = data_of(curl_api("GET", path, bearer=token), "list ops system logs") + rows = items_of(data) + returned += len(rows) + if total is None and isinstance(data, dict) and isinstance(data.get("total"), int): + total = data.get("total") + for row in rows: + if not isinstance(row, dict): + continue + row_id = row.get("id") + key = str(row_id) if row_id is not None else json.dumps(row, sort_keys=True, default=str) + rows_by_id[key] = row + if not rows or len(rows) < 200 or (isinstance(total, int) and returned >= total): + break + page += 1 + except Exception as exc: + return None, { + "status": "unavailable", + "source": "sub2api-native-admin-ops-system-logs", + "reason": compact_error(str(exc)), + "fallback": "runtime-container-logs", + } + queries.append({"marker": marker, "returned": returned, "total": total if total is not None else returned, "pagesRead": page, "truncated": False}) + lines = [] + for row in sorted(rows_by_id.values(), key=lambda item: str(item.get("created_at") or "")): + extra = dict(row.get("extra") or {}) if isinstance(row.get("extra"), dict) else {} + for source_key, target_key in ( + ("request_id", "request_id"), ("client_request_id", "client_request_id"), + ("account_id", "account_id"), ("platform", "platform"), ("model", "model"), + ): + if extra.get(target_key) is None and row.get(source_key) is not None: + extra[target_key] = row.get(source_key) + message = str(row.get("message") or "") + lines.append(str(row.get("created_at") or "") + "\\t" + message + "\\t" + json.dumps(extra, separators=(",", ":"), default=str)) + return lines, { + "status": "available", + "source": "sub2api-native-admin-ops-system-logs", + "uniqueEventCount": len(lines), + "queries": queries, + "fallbackUsed": False, + } + +def policy_effectiveness(lines, accounts, source, group_id=None): + account_names = {str(item.get("id")): item.get("name") for item in accounts} + account_ids = set(account_names.keys()) + account_policy = { + key: { + "accountId": item.get("id"), "accountName": item.get("name"), + "tempUnschedulableEvents": 0, "failoverEvents": 0, "failoverRequestIds": set(), + "failoverRecoveredRequestIds": set(), "failoverFailedRequestIds": set(), "failoverOutcomeMissingRequestIds": set(), + "forwardFailedEvents": 0, "forwardFailedRequestIds": set(), "sameAccountRetryEvents": 0, + "upstreamStatusCodes": {}, + } + for key, item in ((str(item.get("id")), item) for item in accounts) + } + requests = {} + counts = { + "tempUnschedulableEvents": 0, + "failoverEvents": 0, + "selectFailedEvents": 0, + "forwardFailedEvents": 0, + "sameAccountRetryEvents": 0, + "gatewayCompletedEvents": 0, + "gatewayFinalErrorEvents": 0, + } + missing_request_ids = { + "tempUnschedulable": 0, + "failover": 0, + "selectFailed": 0, + "forwardFailed": 0, + "sameAccountRetry": 0, + "gatewayCompleted": 0, + } + temp_accounts = {} + temp_samples = [] + failover_samples = [] + select_failed_samples = [] + final_statuses = {} + event_line_count = 0 + + def request_entry(request_id): + if not isinstance(request_id, str) or not request_id: + return None + if request_id not in requests: + requests[request_id] = { + "requestId": request_id, + "tempUnschedulableCount": 0, + "failoverCount": 0, + "selectFailedCount": 0, + "forwardFailedCount": 0, + "sameAccountRetryCount": 0, + "finalStatus": None, + } + return requests[request_id] + + for line in lines: + if not any(marker in line for marker in ( + "account_temp_unschedulable", "openai.upstream_failover_switching", + "openai.account_select_failed", "openai.forward_failed", + "openai.pool_mode_same_account_retry", "gateway.failover_same_account_retry", + "http request completed", + )): + continue + json_start = line.find("{") + if json_start < 0: + continue + try: + item = json.loads(line[json_start:]) + except json.JSONDecodeError: + continue + if not isinstance(item, dict): + continue + at = line.split("\\t", 1)[0].strip() or None + request_id = item.get("request_id") + account_id = item.get("account_id") + event_group_id = item.get("group_id") + if event_group_id is not None and group_id is not None and str(event_group_id) != str(group_id): + continue + if account_id is not None and str(account_id) not in account_ids: + continue + explicitly_scoped = account_id is not None or event_group_id is not None + if not explicitly_scoped and (not isinstance(request_id, str) or request_id not in requests): + continue + request = request_entry(request_id) + base = { + "at": at, + "requestId": request_id, + "accountId": account_id, + "accountName": account_names.get(str(account_id)), + } + raw_account_stat = account_policy.get(str(account_id)) + account_stat = raw_account_stat if group_id is None or event_group_id is not None else None + if "account_temp_unschedulable" in line: + event_line_count += 1 + counts["tempUnschedulableEvents"] += 1 + if request is not None: + request["tempUnschedulableCount"] += 1 + else: + missing_request_ids["tempUnschedulable"] += 1 + key = str(account_id) if account_id is not None else "unknown" + if key not in temp_accounts: + temp_accounts[key] = { + "accountId": account_id, + "accountName": account_names.get(str(account_id)), + "count": 0, + } + temp_accounts[key]["count"] += 1 + if raw_account_stat is not None: + raw_account_stat["tempUnschedulableEvents"] += 1 + if len(temp_samples) < 20: + temp_samples.append({ + **base, + "statusCode": item.get("status_code"), + "ruleIndex": item.get("rule_index"), + "matchedKeyword": item.get("matched_keyword"), + "until": item.get("until") or item.get("temp_unschedulable_until"), + "reason": compact_error(item.get("reason") or item.get("error")), + }) + elif "openai.upstream_failover_switching" in line: + event_line_count += 1 + counts["failoverEvents"] += 1 + if request is not None: + request["failoverCount"] += 1 + else: + missing_request_ids["failover"] += 1 + if account_stat is not None: + account_stat["failoverEvents"] += 1 + if isinstance(request_id, str) and request_id: + account_stat["failoverRequestIds"].add(request_id) + status_code = item.get("upstream_status") + if isinstance(status_code, int): + key = str(status_code) + account_stat["upstreamStatusCodes"][key] = account_stat["upstreamStatusCodes"].get(key, 0) + 1 + if len(failover_samples) < 20: + failover_samples.append({ + **base, + "switchCount": item.get("switch_count"), + "maxSwitches": item.get("max_switches"), + }) + elif "openai.account_select_failed" in line: + event_line_count += 1 + counts["selectFailedEvents"] += 1 + if request is not None: + request["selectFailedCount"] += 1 + else: + missing_request_ids["selectFailed"] += 1 + if len(select_failed_samples) < 20: + select_failed_samples.append({ + **base, + "excludedAccountCount": item.get("excluded_account_count"), + "error": compact_error(item.get("error")), + }) + elif "openai.forward_failed" in line: + event_line_count += 1 + counts["forwardFailedEvents"] += 1 + if request is not None: + request["forwardFailedCount"] += 1 + else: + missing_request_ids["forwardFailed"] += 1 + if account_stat is not None: + account_stat["forwardFailedEvents"] += 1 + if isinstance(request_id, str) and request_id: + account_stat["forwardFailedRequestIds"].add(request_id) + elif "openai.pool_mode_same_account_retry" in line or "gateway.failover_same_account_retry" in line: + event_line_count += 1 + counts["sameAccountRetryEvents"] += 1 + if request is not None: + request["sameAccountRetryCount"] += 1 + else: + missing_request_ids["sameAccountRetry"] += 1 + if raw_account_stat is not None: + raw_account_stat["sameAccountRetryEvents"] += 1 + elif "http request completed" in line and gateway_request_path(item.get("path")): + event_line_count += 1 + counts["gatewayCompletedEvents"] += 1 + status_code = item.get("status_code") + if request is not None and isinstance(status_code, int): + request["finalStatus"] = status_code + elif request is None: + missing_request_ids["gatewayCompleted"] += 1 + if isinstance(status_code, int): + key = str(status_code) + final_statuses[key] = final_statuses.get(key, 0) + 1 + if status_code >= 400: + counts["gatewayFinalErrorEvents"] += 1 + + correlated = [item for item in requests.values() if item["failoverCount"] > 0] + succeeded = [item for item in correlated if isinstance(item.get("finalStatus"), int) and item["finalStatus"] < 400] + failed = [item for item in correlated if isinstance(item.get("finalStatus"), int) and item["finalStatus"] >= 400] + incomplete = [item for item in correlated if not isinstance(item.get("finalStatus"), int)] + cooled_then_failover = [item for item in correlated if item["tempUnschedulableCount"] > 0] + forward_with_http_success = [ + item for item in requests.values() + if item["forwardFailedCount"] > 0 and isinstance(item.get("finalStatus"), int) and item["finalStatus"] < 400 + ] + forward_with_http_error = [ + item for item in requests.values() + if item["forwardFailedCount"] > 0 and isinstance(item.get("finalStatus"), int) and item["finalStatus"] >= 400 + ] + forward_outcome_missing = [ + item for item in requests.values() + if item["forwardFailedCount"] > 0 and not isinstance(item.get("finalStatus"), int) + ] + for request in correlated: + request_id = request.get("requestId") + if not isinstance(request_id, str): + continue + for current in account_policy.values(): + if request_id not in current["failoverRequestIds"]: + continue + final_status = request.get("finalStatus") + if isinstance(final_status, int) and final_status < 400: + current["failoverRecoveredRequestIds"].add(request_id) + elif isinstance(final_status, int): + current["failoverFailedRequestIds"].add(request_id) + else: + current["failoverOutcomeMissingRequestIds"].add(request_id) + account_policy_rows = [] + for current in account_policy.values(): + account_policy_rows.append({ + "accountId": current["accountId"], "accountName": current["accountName"], + "tempUnschedulableEvents": current["tempUnschedulableEvents"], + "failoverEvents": current["failoverEvents"], "failoverRequests": len(current["failoverRequestIds"]), + "failoverRecovered": len(current["failoverRecoveredRequestIds"]), + "failoverFailed": len(current["failoverFailedRequestIds"]), + "failoverOutcomeMissing": len(current["failoverOutcomeMissingRequestIds"]), + "forwardFailedEvents": current["forwardFailedEvents"], "forwardFailedRequests": len(current["forwardFailedRequestIds"]), + "sameAccountRetryEvents": current["sameAccountRetryEvents"], + "_failureRequestIds": sorted(current["failoverRequestIds"] | current["forwardFailedRequestIds"]), + "upstreamStatusBuckets": [ + {"statusCode": int(code), "count": count} + for code, count in sorted(current["upstreamStatusCodes"].items(), key=lambda pair: int(pair[0])) + ], + }) + sample_limit = 20 if PAYLOAD.get("full") is True else (2 if PAYLOAD.get("account") is not None else 5) + completed = counts["gatewayCompletedEvents"] + final_errors = counts["gatewayFinalErrorEvents"] + result = { + "source": source, + "scope": "group-membership-window", + "attribution": "Events with group_id are exact. Events without group_id are attributed by account membership and request_id correlation; shared-account events can appear in more than one group drilldown.", + "eventLineCount": event_line_count, + **counts, + "tempUnschedulableAccounts": sorted(temp_accounts.values(), key=lambda item: (-item["count"], str(item.get("accountName") or ""))), + "accountPolicyMetrics": sorted(account_policy_rows, key=lambda item: int(item.get("accountId") or 0)), + "accountPolicyAttribution": "Failover and forward-failure request ids enter group scoring only when system-log events carry group_id. Account temp-unschedulable and same-account retry counts are account-wide operational signals and are not added to the failure-rate numerator.", + "failoverRequestCount": len(correlated), + "failoverSucceededRequestCount": len(succeeded), + "failoverFailedRequestCount": len(failed), + "failoverOutcomeMissingRequestCount": len(incomplete), + "cooledThenFailoverRequestCount": len(cooled_then_failover), + "forwardFailureWithHttpSuccessRequestCount": len(forward_with_http_success), + "forwardFailureWithHttpErrorRequestCount": len(forward_with_http_error), + "forwardFailureOutcomeMissingRequestCount": len(forward_outcome_missing), + "requestIdCoverage": { + "missingByEventType": missing_request_ids, + "tempToFailoverCorrelationAvailable": counts["tempUnschedulableEvents"] > missing_request_ids["tempUnschedulable"], + }, + "gatewayFinalStatusBuckets": [ + {"statusCode": int(code), "count": count} + for code, count in sorted(final_statuses.items(), key=lambda pair: int(pair[0])) + ], + "gatewayOutcomeEvidence": { + "completedLogEventCount": completed, + "finalErrorLogEventCount": final_errors, + "observedFinalErrorShare": round(final_errors / completed, 4) if completed > 0 else None, + "denominator": "gateway http request completed log events for Responses, Messages, and Chat Completions paths", + "coverage": "since-window-and-tail-bounded", + "completeTrafficDenominatorGuaranteed": False, + }, + "requestDrilldown": { + "succeededAfterFailover": [item["requestId"] for item in succeeded[:sample_limit]], + "failedAfterFailover": [item["requestId"] for item in failed[:sample_limit]], + "outcomeMissingAfterFailover": [item["requestId"] for item in incomplete[:sample_limit]], + "selectFailed": [item["requestId"] for item in requests.values() if item["selectFailedCount"] > 0][:sample_limit], + "forwardFailureWithHttpSuccess": [item["requestId"] for item in forward_with_http_success[:sample_limit]], + "forwardFailureWithHttpError": [item["requestId"] for item in forward_with_http_error[:sample_limit]], + "forwardFailureOutcomeMissing": [item["requestId"] for item in forward_outcome_missing[:sample_limit]], + }, + "assessment": ( + "failover-observed-with-success" if succeeded else + "failover-observed-without-success" if correlated else + "no-failover-observed" + ), + "disclosure": "Rule matches require account_temp_unschedulable evidence; status codes alone are not treated as rule hits. Request outcomes are joined only by request_id. HTTP success can coexist with forward_failed on streaming responses and does not prove the client received a complete terminal event.", + } + if PAYLOAD.get("full") is True: + result["samples"] = { + "tempUnschedulable": temp_samples, + "failovers": failover_samples, + "selectFailures": select_failed_samples, + } + return result + +def observed_runtime_errors(token, accounts, group_id, platform=None): + selector = PAYLOAD.get("account") + selected = [item for item in accounts if selector is None or str(item.get("id")) == str(selector) or item.get("name") == selector] + if selector is not None and len(selected) != 1: + raise RuntimeError("account-not-found-or-ambiguous: " + str(selector)) + selected_ids = set(str(item.get("id")) for item in selected) + stats = {} + for item in selected: + stats[str(item.get("id"))] = { + "accountName": item.get("name"), + "accountId": item.get("id"), + "upstreamEventCount": 0, + "observedStatusCodes": {}, + "forwardFailureCount": 0, + "forwardFailureStatusCodes": {}, + "lastSeenAt": None, + "errorSamples": [], + } + native_ops = native_ops_snapshot(token, group_id, platform) + lines, system_log_source = native_system_log_lines(token, platform) + if lines is None: + lines = runtime_log_lines() + system_log_source["fallbackUsed"] = True + else: + marker_queries = [item for item in system_log_source.get("queries", []) if item.get("marker") != "http request completed"] + marker_total = sum(item.get("total", 0) for item in marker_queries if isinstance(item.get("total"), int)) + overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) + upstream_count = overview_summary.get("upstream_error_count_excl_429_529") or 0 + upstream_count += overview_summary.get("upstream_429_count") or 0 + upstream_count += overview_summary.get("upstream_529_count") or 0 + if marker_total == 0 and upstream_count > 0: + runtime_event_lines = [ + line for line in runtime_log_lines() + if any(marker in line for marker in ( + "account_temp_unschedulable", "openai.upstream_failover_switching", + "openai.account_select_failed", "openai.forward_failed", + )) + ] + lines.extend(runtime_event_lines) + system_log_source["source"] = "sub2api-native-admin-ops-with-runtime-event-fallback" + system_log_source["fallbackUsed"] = True + system_log_source["fallbackReason"] = "native system-log index returned no policy events while the native overview reported upstream errors" + system_log_source["fallbackEventCount"] = len(runtime_event_lines) + effectiveness = policy_effectiveness(lines, selected if selector is not None else accounts, system_log_source.get("source"), group_id) + effectiveness["sourceStatus"] = system_log_source + overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) + native_ops["responseHeaderTimeout"] = runtime_response_header_timeout() + native_ops["requestTiming"] = native_request_timing(token, group_id, platform, overview_summary) + native_ops["customerErrors"] = native_customer_error_profile(token, group_id, platform) + native_ops["accountQuality"] = account_quality_scores(token, selected, group_id, native_ops, effectiveness) + request_total = overview_summary.get("request_count_total") + error_total = overview_summary.get("error_count_total") + if isinstance(request_total, int) and isinstance(error_total, int): + effectiveness["gatewayOutcomeEvidence"] = { + "source": "sub2api-native-admin-ops-dashboard-overview", + "requestCountTotal": request_total, + "errorCountTotal": error_total, + "errorRate": overview_summary.get("error_rate"), + "upstreamErrorRate": overview_summary.get("upstream_error_rate"), + "businessLimitedCount": overview_summary.get("business_limited_count"), + "startTime": overview_summary.get("start_time"), + "endTime": overview_summary.get("end_time"), + "completeTrafficDenominatorGuaranteed": True, + } + scanned = 0 + for line in lines: + if "account_upstream_error" not in line and "openai.forward_failed" not in line: + continue + json_start = line.find("{") + if json_start < 0: + continue + try: + item = json.loads(line[json_start:]) + except json.JSONDecodeError: + continue + account_id = item.get("account_id") if isinstance(item, dict) else None + account_key = str(account_id) + if account_key not in selected_ids: + continue + scanned += 1 + at = line.split("\t", 1)[0].strip() or None + current = stats[account_key] + if at and (current["lastSeenAt"] is None or at > current["lastSeenAt"]): + current["lastSeenAt"] = at + if "account_upstream_error" in line: + code = item.get("status_code") + if isinstance(code, int): + key = str(code) + current["observedStatusCodes"][key] = current["observedStatusCodes"].get(key, 0) + 1 + current["upstreamEventCount"] += 1 + continue + current["forwardFailureCount"] += 1 + error = compact_error(item.get("error")) + match = re.search(r"(?:upstream error:\\s*|status(?:_code)?[=:]\\s*)(\\d{3})", error, re.IGNORECASE) + if match: + key = match.group(1) + current["forwardFailureStatusCodes"][key] = current["forwardFailureStatusCodes"].get(key, 0) + 1 + if error and error not in current["errorSamples"] and len(current["errorSamples"]) < 3: + current["errorSamples"].append(error) + rows = [] + aggregate = {} + for item in selected: + current = stats[str(item.get("id"))] + observed = [{"statusCode": int(code), "count": count} for code, count in sorted(current["observedStatusCodes"].items(), key=lambda pair: int(pair[0]))] + forwarded = [{"statusCode": int(code), "count": count} for code, count in sorted(current["forwardFailureStatusCodes"].items(), key=lambda pair: int(pair[0]))] + for entry in observed: + key = str(entry["statusCode"]) + aggregate[key] = aggregate.get(key, 0) + entry["count"] + current["observedStatusCodes"] = observed + current["forwardFailureStatusCodes"] = forwarded + analysis = ops_error_analysis(token, item, observed) + has_ops_errors = analysis.get("status") == "available" and (analysis.get("opsRecordCount") or 0) > 0 + if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or has_ops_errors: + if PAYLOAD.get("full") is not True and analysis.get("status") == "available": + analysis = { + "source": analysis.get("source"), + "status": analysis.get("status"), + "opsRecordCount": analysis.get("opsRecordCount"), + "recordsTruncated": analysis.get("recordsTruncated"), + "statusBuckets": analysis.get("statusBuckets"), + "causeBuckets": analysis.get("causeBuckets"), + "rootCause": analysis.get("rootCause"), + "errorRateEvidence": analysis.get("errorRateEvidence"), + "concurrencyRecommendation": analysis.get("concurrencyRecommendation"), + "runtimeConcurrency": analysis.get("runtimeConcurrency"), + "samples": (analysis.get("samples") or [])[:1], + } + current["opsAnalysis"] = analysis + if PAYLOAD.get("full") is not True: + current["errorSamples"] = current["errorSamples"][:1] + if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or has_ops_errors or selector is not None: + rows.append(current) + root_causes = {} + account_roots = [] + for row in rows: + analysis = row.get("opsAnalysis") + if not isinstance(analysis, dict) or analysis.get("status") != "available": + continue + for bucket in analysis.get("causeBuckets") or []: + cause = bucket.get("cause") + count = bucket.get("count") + if isinstance(cause, str) and isinstance(count, int): + root_causes[cause] = root_causes.get(cause, 0) + count + root = analysis.get("rootCause") or {} + account_roots.append({ + "accountId": row.get("accountId"), + "accountName": row.get("accountName"), + "dominantCause": root.get("dominantCause"), + "dominantCount": root.get("dominantCount"), + "dominantShare": root.get("dominantShare"), + }) + ranked_causes = sorted(root_causes.items(), key=lambda pair: (-pair[1], pair[0])) + native_overview = native_ops.get("overview") or {} + native_overview_values = native_overview.get("summary") if isinstance(native_overview.get("summary"), dict) else native_overview + native_error_rate_available = isinstance(native_overview_values.get("error_rate") if isinstance(native_overview_values, dict) else None, (int, float)) + account_count_with_errors = sum( + 1 for item in rows + if item["upstreamEventCount"] > 0 + or item["forwardFailureCount"] > 0 + or (((item.get("opsAnalysis") or {}).get("opsRecordCount") or 0) > 0) + ) + if selector is None and PAYLOAD.get("full") is not True: + for row in rows: + row.pop("opsAnalysis", None) + if PAYLOAD.get("full") is not True: + overview = ((native_ops.get("overview") or {}).get("summary") or {}) + availability = ((native_ops.get("accountAvailability") or {}).get("summary") or {}) + concurrency = ((native_ops.get("concurrency") or {}).get("summary") or {}) + sink = ((native_ops.get("systemLogSink") or {}).get("health") or {}) + unavailable_accounts = availability.get("unavailableAccounts") or [] + if selector is not None: + unavailable_accounts = [item for item in unavailable_accounts if item.get("accountId") in selected_ids] + native_ops = { + "source": native_ops.get("source"), + "overview": { + "status": (native_ops.get("overview") or {}).get("status"), + "healthScore": overview.get("health_score"), + "requestCount": overview.get("request_count_total"), + "errorCount": overview.get("error_count_total"), + "errorRate": overview.get("error_rate"), + "upstreamErrorRate": overview.get("upstream_error_rate"), + "businessLimitedCount": overview.get("business_limited_count"), + }, + "responseHeaderTimeout": native_ops.get("responseHeaderTimeout"), + "requestTiming": native_ops.get("requestTiming"), + "customerErrors": native_ops.get("customerErrors"), + "accountQuality": native_ops.get("accountQuality"), + "accountAvailability": { + "status": (native_ops.get("accountAvailability") or {}).get("status"), + "group": availability.get("group"), + "unavailableAccountCount": availability.get("unavailableAccountCount"), + "unavailableAccounts": unavailable_accounts, + }, + "concurrency": { + "status": (native_ops.get("concurrency") or {}).get("status"), + "group": concurrency.get("group"), + }, + "systemLogSink": { + "status": (native_ops.get("systemLogSink") or {}).get("status"), + "queueDepth": sink.get("queue_depth"), + "droppedCount": sink.get("dropped_count"), + "writeFailedCount": sink.get("write_failed_count"), + }, + } + source_status = effectiveness.get("sourceStatus") or {} + compact_source_status = { + "status": source_status.get("status"), + "source": source_status.get("source"), + "fallbackUsed": source_status.get("fallbackUsed"), + "fallbackReason": source_status.get("fallbackReason"), + "fallbackEventCount": source_status.get("fallbackEventCount"), + "nativeIndexedEventCounts": { + item.get("marker"): item.get("total") + for item in source_status.get("queries", []) + if isinstance(item, dict) + }, + } + drilldown = effectiveness.get("requestDrilldown") or {} + effectiveness = { + "source": effectiveness.get("source"), + "scope": effectiveness.get("scope"), + "attribution": effectiveness.get("attribution"), + "assessment": effectiveness.get("assessment"), + "ruleMatches": { + "tempUnschedulableEvents": effectiveness.get("tempUnschedulableEvents"), + "accounts": effectiveness.get("tempUnschedulableAccounts"), + }, + "failover": { + "events": effectiveness.get("failoverEvents"), + "requests": effectiveness.get("failoverRequestCount"), + "succeeded": effectiveness.get("failoverSucceededRequestCount"), + "failed": effectiveness.get("failoverFailedRequestCount"), + "outcomeMissing": effectiveness.get("failoverOutcomeMissingRequestCount"), + }, + "forwardFailure": { + "events": effectiveness.get("forwardFailedEvents"), + "withHttpSuccess": effectiveness.get("forwardFailureWithHttpSuccessRequestCount"), + "withHttpError": effectiveness.get("forwardFailureWithHttpErrorRequestCount"), + "outcomeMissing": effectiveness.get("forwardFailureOutcomeMissingRequestCount"), + }, + "selectFailedEvents": effectiveness.get("selectFailedEvents"), + "requestIdCoverage": effectiveness.get("requestIdCoverage"), + "gatewayOutcomeEvidence": effectiveness.get("gatewayOutcomeEvidence"), + "requestDrilldown": {key: value for key, value in drilldown.items() if value}, + "sourceStatus": compact_source_status, + "disclosure": effectiveness.get("disclosure"), + } + if selector is not None: + native_ops["accountAvailability"].pop("group", None) + native_ops.pop("concurrency", None) + native_ops.pop("systemLogSink", None) + effectiveness["sourceStatus"] = { + "source": compact_source_status.get("source"), + "fallbackUsed": compact_source_status.get("fallbackUsed"), + "fallbackReason": compact_source_status.get("fallbackReason"), + } + return { + "window": {"since": PAYLOAD.get("since"), "tail": PAYLOAD.get("tail"), "matchedLogEvents": scanned}, + "aggregateObservedStatusCodes": [{"statusCode": int(code), "count": count} for code, count in sorted(aggregate.items(), key=lambda pair: int(pair[0]))], + "accountCountWithErrors": account_count_with_errors, + "rootCauseAnalysis": { + "source": "ops-upstream-errors", + "scope": "account-window-for-group-members", + "attribution": "The upstream-errors endpoint is account scoped. If accounts belong to multiple groups, these cause buckets are shared evidence and must not be added across groups.", + "causeBuckets": [{"cause": cause, "count": count} for cause, count in ranked_causes], + "dominantCause": ranked_causes[0][0] if ranked_causes else None, + "dominantCount": ranked_causes[0][1] if ranked_causes else 0, + "accountRoots": account_roots, + "errorRateAvailable": native_error_rate_available, + }, + "nativeOps": native_ops, + "policyEffectiveness": effectiveness, + **( + {"accounts": rows} + if selector is not None or PAYLOAD.get("full") is True + else {"accountDrilldown": "rerun runtime errors with --account for per-account evidence"} + ), + "message": "observedStatusCodes comes only from account_upstream_error; forward failures are supplemental message evidence", + } + +def selected_account_details(token, accounts): + selectors = PAYLOAD.get("accounts") + if not isinstance(selectors, list) or not selectors: + selector = PAYLOAD.get("account") + selectors = [selector] if isinstance(selector, str) and selector else [] + details = [] + selected_ids = set() + for raw_selector in selectors: + selector = str(raw_selector) + detail = find_account(token, accounts, selector) + account_id = detail.get("id") + if account_id in selected_ids: + raise RuntimeError("duplicate-account-selector: " + selector) + if PAYLOAD.get("kind") == "temp-unschedulable" and detail.get("type") != "apikey": + raise RuntimeError("runtime temp-unschedulable policy requires an apikey account: " + selector) + selected_ids.add(account_id) + details.append(detail) + return details + +def mutation_plan(detail, action, include_rules): + if PAYLOAD.get("kind") == "priority": + before_priority = int(detail.get("priority") or 0) + desired_priority = int(PAYLOAD.get("priority")) + return { + "accountName": detail.get("name"), + "accountId": detail.get("id"), + "change": "noop" if before_priority == desired_priority else "update", + "before": {"priority": before_priority}, + "desired": {"priority": desired_priority}, + "template": None, + "_kind": "priority", + "_desiredPriority": desired_priority, + } + credentials = dict(detail.get("credentials") or {}) + before = normalize_policy(credentials) + if action == "apply": + template = PAYLOAD.get("selectedTemplate") + if not isinstance(template, dict): + raise RuntimeError("selected runtime template missing") + desired = dict(credentials) + desired.update(template.get("credentials") or {}) + desired_policy = normalize_policy(desired) + exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials + change = "noop" if before == desired_policy else ("update" if exists else "create") + else: + desired = dict(credentials) + desired.pop("temp_unschedulable_enabled", None) + desired.pop("temp_unschedulable_rules", None) + desired_policy = normalize_policy(desired) + change = "delete" if before != desired_policy or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop" + return { + "accountName": detail.get("name"), + "accountId": detail.get("id"), + "change": change, + "before": policy_summary(before, include_rules), + "desired": policy_summary(desired_policy, include_rules), + "template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None, + "_desiredCredentials": desired, + "_desiredPolicy": desired_policy, + "_kind": "temp-unschedulable", + } + +def write_mutation_plan(token, plan): + if plan["change"] == "noop": + return {"attempted": False, "status": "noop", "error": None} + try: + payload = {"priority": plan["_desiredPriority"]} if plan["_kind"] == "priority" else {"credentials": plan["_desiredCredentials"]} + data_of(curl_api("PUT", f"/api/v1/admin/accounts/{plan['accountId']}", bearer=token, payload=payload), "update account runtime config") + return {"attempted": True, "status": "succeeded", "error": None} + except Exception as exc: + return {"attempted": True, "status": "failed", "error": compact_error(str(exc))} + +def reconcile_mutation_plan(token, plan, write, include_rules): + item = {key: value for key, value in plan.items() if not key.startswith("_")} + item["write"] = write + try: + actual_detail = account_detail(token, plan["accountId"]) + if plan["_kind"] == "priority": + actual_priority = int(actual_detail.get("priority") or 0) + item["actual"] = {"priority": actual_priority} + item["reconciled"] = actual_priority == plan["_desiredPriority"] + else: + actual_policy = normalize_policy(actual_detail.get("credentials")) + item["actual"] = policy_summary(actual_policy, include_rules) + item["reconciled"] = actual_policy == plan["_desiredPolicy"] + item["reconcileError"] = None + except Exception as exc: + item["actual"] = None + item["reconciled"] = False + item["reconcileError"] = compact_error(str(exc)) + return item + +def runtime_result(): + token = login() + action = PAYLOAD["action"] + if action == "errors" and PAYLOAD.get("allGroups") is True: + groups = list_groups(token, PAYLOAD.get("platform")) + overview = all_group_error_overview(token, groups) + return { + "ok": True, + "operation": "errors", + "mutation": False, + "endpoint": "host-docker" if RUNTIME_MODE == "host-docker" else "k3s-service", + "allGroups": overview, + "valuesPrinted": False, + } + group_selector = PAYLOAD.get("group") + group_platform = PAYLOAD.get("platform") if action in ("errors", "infrastructure") else "openai" + if action in ("errors", "infrastructure") and group_selector is None: + group_platform = "openai" + groups = list_groups(token, group_platform) + selector = str(group_selector if group_selector is not None else PAYLOAD["groupName"]) + matching_groups = [item for item in groups if str(item.get("id")) == selector or item.get("name") == selector] + if len(matching_groups) > 1: + raise RuntimeError("group-selector-ambiguous: " + selector) + group = matching_groups[0] if matching_groups else None + if not isinstance(group, dict) or group.get("id") is None: + raise RuntimeError("group-not-found: " + selector) + effective_platform = group_platform or group.get("platform") + accounts = list_group_accounts(token, group["id"], effective_platform) + base = { + "group": {"id": group.get("id"), "name": group.get("name"), "platform": effective_platform, "status": group.get("status")}, + "templates": [template_summary(item) for item in PAYLOAD["templates"]], + "endpoint": "host-docker" if RUNTIME_MODE == "host-docker" else "k3s-service", + "valuesPrinted": False, + } + if action == "list": + details = [account_detail(token, item["id"]) for item in accounts] + return {**base, "ok": True, "operation": "list", "mutation": False, "accountCount": len(details), "accounts": [account_summary(item) for item in sorted(details, key=lambda value: value.get("name") or "")]} + if action == "errors": + errors = observed_runtime_errors(token, accounts, group["id"], effective_platform) + return {**base, "ok": True, "operation": "errors", "mutation": False, "errors": errors} + detail = find_account(token, accounts, str(PAYLOAD["account"])) if action in ("get", "infrastructure") else None + if action == "get": + return {**base, "ok": True, "operation": "get", "mutation": False, "account": account_summary(detail, True)} + if action == "infrastructure": + return { + **base, + "ok": True, + "operation": "infrastructure", + "mutation": False, + "account": account_summary(detail, False), + "infrastructure": infrastructure_snapshot(token, detail), + } + include_plan_rules = PAYLOAD.get("full") is True + details = selected_account_details(token, accounts) + plans = [mutation_plan(item, action, include_plan_rules) for item in details] + changed = sum(1 for plan in plans if plan["change"] != "noop") + if PAYLOAD.get("confirm") is not True: + items = [] + for plan in plans: + item = {key: value for key, value in plan.items() if not key.startswith("_")} + item["write"] = {"attempted": False, "status": "dry-run", "error": None} + item["actual"] = item["before"] + item["reconciled"] = None + item["reconcileError"] = None + items.append(item) + summary = {"selected": len(items), "changed": changed, "writeSucceeded": 0, "writeFailed": 0, "reconciled": 0, "mismatched": 0} + return {**base, "ok": True, "operation": action, "mode": "dry-run", "mutation": False, "partialWrite": False, "summary": summary, "items": items, "credentialsPrinted": False} + writes = [write_mutation_plan(token, plan) for plan in plans] + items = [reconcile_mutation_plan(token, plan, write, include_plan_rules) for plan, write in zip(plans, writes)] + write_succeeded = sum(1 for item in items if item["write"]["status"] == "succeeded") + write_failed = sum(1 for item in items if item["write"]["status"] == "failed") + reconciled = sum(1 for item in items if item["reconciled"] is True) + mismatched = sum(1 for item in items if item["reconciled"] is False) + summary = {"selected": len(items), "changed": changed, "writeSucceeded": write_succeeded, "writeFailed": write_failed, "reconciled": reconciled, "mismatched": mismatched} + ok = write_failed == 0 and mismatched == 0 + return {**base, "ok": ok, "operation": action, "mode": "confirmed", "mutation": write_succeeded > 0, "partialWrite": write_succeeded > 0 and write_failed > 0, "summary": summary, "items": items, "credentialsPrinted": False} + +try: + result = runtime_result() +except Exception as exc: + result = {"ok": False, "operation": PAYLOAD.get("action"), "mutation": False, "error": str(exc), "valuesPrinted": False} + +print(json.dumps(result, ensure_ascii=False, indent=2)) +sys.exit(0 if result.get("ok") else 1) +PY +`; +} diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-remote-script.ts b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script.ts new file mode 100644 index 00000000..c279f161 --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/runtime-remote-script.ts @@ -0,0 +1,7 @@ +import type { RuntimeRemoteTarget } from "./runtime-remote-script-head"; +import { runtimeRemoteScriptHead } from "./runtime-remote-script-head"; +import { runtimeRemoteScriptTail } from "./runtime-remote-script-tail"; + +export function runtimeScript(payload: unknown, target: RuntimeRemoteTarget): string { + return runtimeRemoteScriptHead(payload, target) + runtimeRemoteScriptTail(); +} diff --git a/scripts/src/platform-infra-sub2api-codex/runtime-render.ts b/scripts/src/platform-infra-sub2api-codex/runtime-render.ts new file mode 100644 index 00000000..817f4f2b --- /dev/null +++ b/scripts/src/platform-infra-sub2api-codex/runtime-render.ts @@ -0,0 +1,659 @@ +import type { RenderedCliResult } from "../output"; +import { renderedCliResult, renderTable } from "./render"; +import type { RuntimeOptions } from "./runtime-options"; + +export function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown { + if (options.action !== "errors" || options.full || value === null || typeof value !== "object" || Array.isArray(value)) return value; + const runtime = value as Record; + const allGroups = runtimeRecord(runtime.allGroups); + if (allGroups !== null) { + const nextPageToken = typeof allGroups.nextPageToken === "string" ? allGroups.nextPageToken : null; + const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; + const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; + return { + ok: runtime.ok, + operation: runtime.operation, + mutation: runtime.mutation, + endpoint: runtime.endpoint, + allGroups, + disclosure: { + group: `${baseCommand} --group ${platformFlag}`, + nextPage: nextPageToken === null ? null : `${baseCommand} --all-groups${platformFlag} --page-token ${nextPageToken}`, + note: "The overview uses stable group-id pagination. Use group drilldown for account root causes and request indexes.", + }, + valuesPrinted: false, + }; + } + const errors = runtimeRecord(runtime.errors); + if (errors === null) return value; + const effectiveness = runtimeRecord(errors.policyEffectiveness) ?? {}; + const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; + const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; + const availabilityGroup = runtimeRecord(availability.group) ?? {}; + const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; + const concurrencyGroup = runtimeRecord(concurrency.group) ?? {}; + const sink = runtimeRecord(nativeOps.systemLogSink) ?? {}; + const sourceStatus = runtimeRecord(effectiveness.sourceStatus) ?? {}; + const drilldown = runtimeRecord(effectiveness.requestDrilldown) ?? {}; + const requestIndexes = Object.fromEntries(Object.entries(drilldown).flatMap(([key, item]) => { + if (!Array.isArray(item)) return []; + const ids = item.filter((entry): entry is string => typeof entry === "string"); + if (ids.length === 0) return []; + return [[key, { + count: ids.length, + sampleRequestId: ids[0], + moreAvailable: ids.length > 1, + }]]; + })); + const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; + const groupFlag = options.group === null ? "" : ` --group ${options.group}`; + const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; + if (options.json && options.account === null) { + return { + ok: runtime.ok, + operation: runtime.operation, + mutation: runtime.mutation, + group: runtime.group, + endpoint: runtime.endpoint, + errors: { + window: errors.window, + nativeOps: { + source: nativeOps.source, + overview: nativeOps.overview, + accountQuality: compactAccountQuality(nativeOps.accountQuality, false), + accountAvailability: { + status: availability.status, + totalAccounts: availabilityGroup.total_accounts, + availableCount: availabilityGroup.available_count, + rateLimitCount: availabilityGroup.rate_limit_count, + errorCount: availabilityGroup.error_count, + unavailableAccounts: Array.isArray(availability.unavailableAccounts) + ? availability.unavailableAccounts.map((item) => { + const account = runtimeRecord(item) ?? {}; + return { + accountId: account.accountId, + accountName: account.accountName, + status: account.status, + reason: account.error, + }; + }) + : [], + }, + concurrency: { + status: concurrency.status, + currentInUse: concurrencyGroup.current_in_use, + maxCapacity: concurrencyGroup.max_capacity, + loadPercentage: concurrencyGroup.load_percentage, + waitingInQueue: concurrencyGroup.waiting_in_queue, + }, + }, + }, + disclosure: { + account: `${baseCommand}${groupFlag}${platformFlag} --account `, + trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", + note: "This group overview keeps every account name and quality summary. Use account drilldown for request indexes and complete failure evidence.", + }, + valuesPrinted: false, + }; + } + return { + ok: runtime.ok, + operation: runtime.operation, + mutation: runtime.mutation, + group: runtime.group, + endpoint: runtime.endpoint, + errors: { + window: errors.window, + nativeOps: { + source: nativeOps.source, + overview: nativeOps.overview, + responseHeaderTimeout: options.account === null || !options.json ? nativeOps.responseHeaderTimeout : undefined, + requestTiming: options.account === null || !options.json ? compactRequestTiming(nativeOps.requestTiming) : undefined, + customerErrors: compactCustomerErrors(nativeOps.customerErrors, options.account !== null || !options.json), + accountQuality: compactAccountQuality(nativeOps.accountQuality, options.account !== null || !options.json), + accountAvailability: { + status: availability.status, + totalAccounts: availabilityGroup.total_accounts, + availableCount: availabilityGroup.available_count, + rateLimitCount: availabilityGroup.rate_limit_count, + errorCount: availabilityGroup.error_count, + unavailableAccounts: Array.isArray(availability.unavailableAccounts) + ? availability.unavailableAccounts.map((item) => { + const account = runtimeRecord(item) ?? {}; + return { + accountId: account.accountId, + accountName: account.accountName, + status: account.status, + reason: account.error, + }; + }) + : [], + }, + concurrency: { + status: concurrency.status, + currentInUse: concurrencyGroup.current_in_use, + maxCapacity: concurrencyGroup.max_capacity, + loadPercentage: concurrencyGroup.load_percentage, + waitingInQueue: concurrencyGroup.waiting_in_queue, + }, + systemLogSink: { + status: sink.status, + queueDepth: sink.queueDepth, + droppedCount: sink.droppedCount, + writeFailedCount: sink.writeFailedCount, + }, + }, + rootCauseAnalysis: errors.rootCauseAnalysis, + policyEffectiveness: { + source: effectiveness.source, + scope: effectiveness.scope, + attribution: effectiveness.attribution, + assessment: effectiveness.assessment, + ruleMatches: effectiveness.ruleMatches, + failover: effectiveness.failover, + forwardFailure: effectiveness.forwardFailure, + selectFailedEvents: effectiveness.selectFailedEvents, + requestIndexes, + sourceStatus: { + status: sourceStatus.status, + fallbackUsed: sourceStatus.fallbackUsed, + fallbackReason: sourceStatus.fallbackReason, + }, + }, + }, + disclosure: { + complete: `${baseCommand}${groupFlag}${platformFlag}${options.account === null ? "" : " --account "} --full`, + account: `${baseCommand}${groupFlag}${platformFlag} --account `, + trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", + note: "Use requestIndexes for trace drilldown and --full for the complete finite index.", + }, + valuesPrinted: false, + }; +} +function compactRequestTiming(value: unknown): unknown { + const timing = runtimeRecord(value); + if (timing === null) return value; + return { + status: timing.status, + reason: timing.reason, + source: timing.source, + ttft: timing.ttft, + duration: timing.duration, + observedFloorSeconds: timing.observedFloorSeconds, + attribution: timing.attribution, + }; +} +function compactCustomerErrors(value: unknown, detailed: boolean): unknown { + const profile = runtimeRecord(value); + if (profile === null) return value; + return { + status: profile.status, + reason: profile.reason, + source: profile.source, + total: profile.total, + modelBuckets: runtimeRecords(profile.modelBuckets).slice(0, 5), + accountBuckets: detailed ? runtimeRecords(profile.accountBuckets).slice(0, 5) : undefined, + statusBuckets: runtimeRecords(profile.statusBuckets).slice(0, 5), + streamBuckets: runtimeRecords(profile.streamBuckets).slice(0, 5), + messageBuckets: runtimeRecords(profile.messageBuckets).slice(0, detailed ? 10 : 5), + phaseBuckets: detailed ? runtimeRecords(profile.phaseBuckets).slice(0, 5) : undefined, + endpointBuckets: detailed ? runtimeRecords(profile.endpointBuckets).slice(0, 5) : undefined, + pathBuckets: detailed ? runtimeRecords(profile.pathBuckets).slice(0, 5) : undefined, + samples: detailed ? runtimeRecords(profile.samples).slice(0, 15) : undefined, + }; +} +function compactAccountQuality(value: unknown, detailed: boolean): unknown { + const quality = runtimeRecord(value); + if (quality === null) return value; + return { + source: quality.source, + scope: quality.scope, + window: quality.window, + scoreRule: (() => { + const rule = runtimeRecord(quality.scoreRule) ?? {}; + return detailed ? { + reliabilityWeight: rule.reliabilityWeight, + latencyWeight: rule.latencyWeight, + availabilityWeight: rule.availabilityWeight, + failureDefinition: rule.failureDefinition, + missingMetric: rule.missingMetric, + } : { + reliabilityWeight: rule.reliabilityWeight, + latencyWeight: rule.latencyWeight, + availabilityWeight: rule.availabilityWeight, + }; + })(), + accounts: runtimeRecords(quality.accounts).map((account) => detailed ? { + accountId: account.accountId, + accountName: account.accountName, + status: account.status, + currentlyAvailable: account.currentlyAvailable, + score: account.score, + grade: account.grade, + assessment: account.assessment, + confidence: account.confidence, + observedAttempts: account.observedAttempts, + successRequests: account.successRequests, + failureRequests: account.failureRequests, + failureRate: account.failureRate, + firstTokenSamples: account.firstTokenSamples, + firstTokenCoverage: account.firstTokenCoverage, + ttftP95Ms: account.ttftP95Ms, + customerErrorRequests: account.customerErrorRequests, + scoreableUpstreamErrorRequests: account.scoreableUpstreamErrorRequests, + excludedNonUpstreamErrorRequests: account.excludedNonUpstreamErrorRequests, + excludedReasonBuckets: account.excludedReasonBuckets, + usage: account.usage, + failoverRequests: account.failoverRequests, + failoverRecovered: account.failoverRecovered, + failoverFailed: account.failoverFailed, + sameAccountRetryEvents: account.sameAccountRetryEvents, + tempUnschedulableEvents: account.tempUnschedulableEvents, + forwardFailedRequests: account.forwardFailedRequests, + reasons: account.reasons, + } : { + accountId: account.accountId, + accountName: account.accountName, + currentlyAvailable: account.currentlyAvailable, + score: account.score, + grade: account.grade, + confidence: account.confidence, + observedAttempts: account.observedAttempts, + failureRate: account.failureRate, + ttftP95Ms: account.ttftP95Ms, + customerErrorRequests: account.customerErrorRequests, + scoreableUpstreamErrorRequests: account.scoreableUpstreamErrorRequests, + excludedNonUpstreamErrorRequests: account.excludedNonUpstreamErrorRequests, + excludedReasonBuckets: account.excludedReasonBuckets, + usage: account.usage, + failoverRequests: account.failoverRequests, + failoverRecovered: account.failoverRecovered, + failoverFailed: account.failoverFailed, + }), + attribution: detailed ? quality.attribution : undefined, + }; +} +function runtimeRecord(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : null; +} + +function runtimeRecords(value: unknown): Record[] { + return Array.isArray(value) ? value.flatMap((item) => { + const record = runtimeRecord(item); + return record === null ? [] : [record]; + }) : []; +} +function runtimeText(value: unknown): string { + if (value === null || value === undefined || value === "") return "-"; + if (typeof value === "boolean") return value ? "true" : "false"; + if (Array.isArray(value)) return value.map(runtimeText).join(","); + return String(value).replace(/\s+/gu, " ").trim(); +} + +function runtimeShort(value: unknown, max = 42): string { + const text = runtimeText(value); + return text.length <= max ? text : `${text.slice(0, max - 1)}…`; +} + +function runtimeRate(value: unknown): string { + return typeof value === "number" && Number.isFinite(value) ? `${(value * 100).toFixed(2)}%` : "-"; +} + +function runtimeMillions(value: unknown): string { + return typeof value === "number" && Number.isFinite(value) ? `${(value / 1_000_000).toFixed(3)}M` : "-"; +} + +function runtimeDecimal(value: unknown, digits = 4): string { + return typeof value === "number" && Number.isFinite(value) ? value.toFixed(digits) : "-"; +} + +export function renderRuntimeResult(response: Record, options: RuntimeOptions): RenderedCliResult { + const runtime = runtimeRecord(response.runtime) ?? {}; + const ok = response.ok === true; + const lines: string[] = []; + if (options.action === "errors") renderRuntimeErrors(lines, runtime, options); + else if (options.action === "infrastructure") renderRuntimeInfrastructure(lines, runtime); + else if (options.action === "list") renderRuntimeAccountList(lines, runtime); + else if (options.action === "get") renderRuntimeAccountDetail(lines, runtime); + else renderRuntimeMutation(lines, runtime, options); + lines.push("", `JSON: bun scripts/cli.ts platform-infra sub2api codex-pool runtime ${options.action} --json`); + return renderedCliResult(ok, `platform-infra sub2api codex-pool runtime ${options.action}`, lines.join("\n")); +} + +function renderRuntimeErrors(lines: string[], runtime: Record, options: RuntimeOptions): void { + const allGroups = runtimeRecord(runtime.allGroups); + if (allGroups !== null) { + const totals = runtimeRecord(allGroups.pageTotals) ?? {}; + const timeout = runtimeRecord(allGroups.responseHeaderTimeout) ?? {}; + const timing = runtimeRecord(allGroups.requestTiming) ?? {}; + const groups = runtimeRecords(allGroups.groups); + lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=all-groups since=${options.since}`); + lines.push(renderTable([ + ["GROUPS", "RETURNED", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UPSTREAM_ERR", "UP_RATE", "ATTENTION"], + [ + runtimeText(allGroups.totalGroups), runtimeText(allGroups.returnedGroups), runtimeText(totals.requestCount), + runtimeText(totals.errorCount), runtimeRate(totals.errorRate), runtimeText(totals.upstreamErrorCount), + runtimeRate(totals.upstreamErrorRate), runtimeText(totals.groupsNeedingAttention), + ], + ])); + lines.push(renderTable([ + ["OPENAI_HEADER_TIMEOUT", "SOURCE", "GENERIC_TIMEOUT", "GENERIC_APPLIES_TO_OPENAI", "PAGE_MAX_TTFT_P99_MS", "OBSERVED_FLOOR_SECONDS"], + [ + timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source), + `${runtimeText(timeout.genericResponseHeaderTimeoutSeconds)}s`, runtimeText(timeout.genericAppliesToOpenAI), + runtimeText(timing.maxTtftP99Ms), runtimeText(timing.observedFloorSeconds), + ], + ])); + if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); + lines.push("", "GROUPS"); + lines.push(renderTable([ + ["NAME", "ID", "PLATFORM", "STATUS", "REQ", "CLIENT_ERR", "ERR_RATE", "UP_ERR", "UP_RATE", "TTFT_P99_MS", "DUR_P99_MS", "ACCOUNTS", "AVAILABLE", "USE/CAP", "QUEUE", "ATTN"], + ...groups.map((group) => [ + runtimeShort(group.groupName, 28), runtimeText(group.groupId), runtimeText(group.platform), runtimeText(group.status), + runtimeText(group.requestCount), runtimeText(group.errorCount), runtimeRate(group.errorRate), runtimeText(group.upstreamErrorCount), + runtimeRate(group.upstreamErrorRate), runtimeText(group.ttftP99Ms), runtimeText(group.durationP99Ms), runtimeText(group.totalAccounts), runtimeText(group.availableCount), + `${runtimeText(group.currentInUse)}/${runtimeText(group.maxCapacity)}`, runtimeText(group.waitingInQueue), group.needsAttention === true ? "yes" : "no", + ]), + ])); + const disclosure = runtimeRecord(runtime.disclosure) ?? {}; + lines.push("", `Detail: ${runtimeText(disclosure.group)}`); + if (typeof disclosure.nextPage === "string") lines.push(`Next: ${disclosure.nextPage}`); + return; + } + const group = runtimeRecord(runtime.group) ?? {}; + const errors = runtimeRecord(runtime.errors) ?? {}; + const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; + const overview = runtimeRecord(nativeOps.overview) ?? {}; + const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; + const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; + const timeout = runtimeRecord(nativeOps.responseHeaderTimeout) ?? {}; + const timing = runtimeRecord(nativeOps.requestTiming) ?? {}; + const ttft = runtimeRecord(timing.ttft) ?? {}; + const duration = runtimeRecord(timing.duration) ?? {}; + const customerErrors = runtimeRecord(nativeOps.customerErrors) ?? {}; + const accountQuality = runtimeRecord(nativeOps.accountQuality) ?? {}; + lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`); + lines.push(renderTable([ + ["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"], + [ + runtimeShort(group.name, 30), runtimeText(group.id), runtimeText(group.platform), runtimeText(overview.requestCount), + runtimeText(overview.errorCount), runtimeRate(overview.errorRate), runtimeRate(overview.upstreamErrorRate), + runtimeText(overview.businessLimitedCount), runtimeText(overview.healthScore), + ], + ])); + lines.push(renderTable([ + ["OPENAI_HEADER_TIMEOUT", "SOURCE", "TTFT_P50_MS", "TTFT_P95_MS", "TTFT_P99_MS", "TTFT_MAX_MS", "DURATION_P99_MS", "DURATION_MAX_MS"], + [ + timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source), + runtimeText(ttft.p50_ms), runtimeText(ttft.p95_ms), runtimeText(ttft.p99_ms), runtimeText(ttft.max_ms), + runtimeText(duration.p99_ms), runtimeText(duration.max_ms), + ], + ])); + const candidates = runtimeRecords(timing.candidates); + if (candidates.length > 0) lines.push("", "TIMEOUT CANDIDATES", renderTable([ + ["SECONDS", "OBSERVED_FLOOR", "TTFT_P99_HEADROOM_MS", "TTFT_P99_BELOW", "ALL_DUR>=", "ERROR_DUR>=", "ASSESSMENT"], + ...candidates.map((candidate) => [ + runtimeText(candidate.seconds), runtimeText(candidate.observedFloor), runtimeText(candidate.ttftP99HeadroomMs), runtimeText(candidate.ttftP99Below), + runtimeText(candidate.totalDurationAtLeastCount), runtimeText(candidate.errorDurationAtLeastCount), runtimeText(candidate.assessment), + ]), + ])); + if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); + const modelTiming = runtimeRecords(timing.modelTiming); + if (modelTiming.length > 0) lines.push("", "MODEL TIMING", renderTable([ + ["MODEL", "REQUESTS", "FIRST_TOKEN_ROWS", "COVERAGE", "AVG_TTFT_MS", "AVG_DURATION_MS", "AVG_TOKENS_S"], + ...modelTiming.map((item) => [runtimeShort(item.model, 32), runtimeText(item.requestCount), runtimeText(item.requestsWithFirstToken), runtimeRate(item.firstTokenCoverage), runtimeText(item.avgFirstTokenMs), runtimeText(item.avgDurationMs), runtimeText(item.avgTokensPerSec)]), + ])); + const qualityAccounts = runtimeRecords(accountQuality.accounts); + if (qualityAccounts.length > 0) { + lines.push("", "ACCOUNT QUALITY", renderTable([ + ["ACCOUNT", "ID", "SCORE", "GRADE", "QUALITY", "CONF", "ATTEMPTS", "SUCCESS", "FAIL", "UP_CUST", "EXCLUDED", "FAIL_RATE", "TTFT_P95_MS", "TTFT_N", "AVAILABLE", "REASONS"], + ...qualityAccounts.map((item) => [ + runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.score), runtimeText(item.grade), runtimeText(item.assessment), runtimeText(item.confidence), + runtimeText(item.observedAttempts), runtimeText(item.successRequests), runtimeText(item.failureRequests), runtimeText(item.scoreableUpstreamErrorRequests), + runtimeText(item.excludedNonUpstreamErrorRequests), runtimeRate(item.failureRate), + runtimeText(item.ttftP95Ms), runtimeText(item.firstTokenSamples), runtimeText(item.currentlyAvailable), + runtimeShort(Array.isArray(item.reasons) ? item.reasons.join(",") : "", 58), + ]), + ])); + lines.push("ACCOUNT USAGE AND COST", renderTable([ + ["ACCOUNT", "ID", "REQUESTS", "TOKENS", "API_USD", "COST_RATE_CNY", "UPSTREAM_COST_CNY", "EXCLUDED_REASONS"], + ...qualityAccounts.map((item) => { + const usage = runtimeRecord(item.usage) ?? {}; + const excluded = runtimeRecords(item.excludedReasonBuckets).map((entry) => `${runtimeText(entry.reason)}:${runtimeText(entry.count)}`).join(","); + return [ + runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(usage.requestCount), runtimeMillions(usage.tokenCount), + runtimeDecimal(usage.apiAmountUsd, 4), runtimeDecimal(usage.costRateCnyPerApiUsd, 3), runtimeDecimal(usage.upstreamCostCny, 4), + runtimeShort(excluded, 64), + ]; + }), + ])); + const controlRows = qualityAccounts.filter((item) => [ + item.failoverRequests, item.sameAccountRetryEvents, item.tempUnschedulableEvents, item.forwardFailedRequests, item.customerErrorRequests, + ].some((value) => typeof value === "number" && value > 0)); + if (controlRows.length > 0) lines.push("ACCOUNT FAILURE CONTROL", renderTable([ + ["ACCOUNT", "ID", "FAILOVER", "RECOVERED", "FAILED", "MISSING", "RETRY", "TEMP_UNSCHED", "FORWARD_FAIL", "CUSTOMER_ERR"], + ...controlRows.map((item) => [ + runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.failoverRequests), runtimeText(item.failoverRecovered), + runtimeText(item.failoverFailed), runtimeText(item.failoverOutcomeMissing), runtimeText(item.sameAccountRetryEvents), + runtimeText(item.tempUnschedulableEvents), runtimeText(item.forwardFailedRequests), runtimeText(item.customerErrorRequests), + ]), + ])); + const scoreRule = runtimeRecord(accountQuality.scoreRule) ?? {}; + lines.push(`SCORE RULE reliability=${runtimeText(scoreRule.reliabilityWeight)} latency=${runtimeText(scoreRule.latencyWeight)} availability=${runtimeText(scoreRule.availabilityWeight)} confidence=separate`); + if (typeof accountQuality.attribution === "string") lines.push(`NOTE: ${accountQuality.attribution}`); + } + const errorModels = runtimeRecords(customerErrors.modelBuckets); + const errorAccounts = runtimeRecords(customerErrors.accountBuckets); + const errorDimensions = [ + ...runtimeRecords(customerErrors.statusBuckets).map((item) => ["status", runtimeText(item.value), runtimeText(item.count)]), + ...runtimeRecords(customerErrors.streamBuckets).map((item) => ["mode", runtimeText(item.value), runtimeText(item.count)]), + ...runtimeRecords(customerErrors.phaseBuckets).map((item) => ["phase", runtimeText(item.value), runtimeText(item.count)]), + ...runtimeRecords(customerErrors.endpointBuckets).map((item) => ["endpoint", runtimeText(item.value), runtimeText(item.count)]), + ...runtimeRecords(customerErrors.pathBuckets).map((item) => ["path", runtimeText(item.value), runtimeText(item.count)]), + ]; + if (errorModels.length > 0) lines.push("", `CUSTOMER ERRORS source=${runtimeText(customerErrors.source)} total=${runtimeText(customerErrors.total)}`, renderTable([ + ["MODEL", "COUNT"], ...errorModels.map((item) => [runtimeShort(item.value, 40), runtimeText(item.count)]), + ])); + if (errorAccounts.length > 0) lines.push(renderTable([ + ["ACCOUNT_ID", "COUNT"], ...errorAccounts.map((item) => [runtimeText(item.value), runtimeText(item.count)]), + ])); + const errorMessages = runtimeRecords(customerErrors.messageBuckets); + if (errorMessages.length > 0) lines.push("CUSTOMER ERROR MESSAGES", renderTable([ + ["MESSAGE", "COUNT"], ...errorMessages.map((item) => [runtimeShort(item.value, 100), runtimeText(item.count)]), + ])); + if (errorDimensions.length > 0) lines.push(renderTable([["DIMENSION", "VALUE", "COUNT"], ...errorDimensions])); + const errorSamples = runtimeRecords(customerErrors.samples); + if (errorSamples.length > 0) lines.push("CUSTOMER ERROR INDEX", renderTable([ + ["REQUEST_ID", "MODEL", "STATUS", "ACCOUNT", "MODE", "PATH", "MESSAGE"], + ...errorSamples.map((item) => [runtimeText(item.requestId), runtimeShort(item.model, 26), runtimeText(item.statusCode), runtimeText(item.accountId), runtimeText(item.mode), runtimeShort(item.path, 34), runtimeShort(item.message, 48)]), + ])); + lines.push(renderTable([ + ["ACCOUNTS", "AVAILABLE", "RATE_LIMIT", "ERROR", "IN_USE", "CAPACITY", "QUEUE"], + [ + runtimeText(availability.totalAccounts), runtimeText(availability.availableCount), runtimeText(availability.rateLimitCount), + runtimeText(availability.errorCount), runtimeText(concurrency.currentInUse), runtimeText(concurrency.maxCapacity), runtimeText(concurrency.waitingInQueue), + ], + ])); + const unavailable = runtimeRecords(availability.unavailableAccounts); + if (unavailable.length > 0) { + lines.push("", "UNAVAILABLE ACCOUNTS", renderTable([ + ["ACCOUNT", "ID", "STATUS", "REASON"], + ...unavailable.map((account) => [runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.status), runtimeShort(account.reason, 70)]), + ])); + } + const root = runtimeRecord(errors.rootCauseAnalysis) ?? {}; + const causes = runtimeRecords(root.causeBuckets); + if (causes.length > 0) { + lines.push("", `ROOT CAUSES scope=${runtimeText(root.scope)}`, renderTable([ + ["CAUSE", "COUNT"], + ...causes.map((cause) => [runtimeText(cause.cause), runtimeText(cause.count)]), + ])); + } + const policy = runtimeRecord(errors.policyEffectiveness) ?? {}; + const rules = runtimeRecord(policy.ruleMatches) ?? {}; + const failover = runtimeRecord(policy.failover) ?? {}; + const forward = runtimeRecord(policy.forwardFailure) ?? {}; + lines.push("", `POLICY EFFECTIVENESS scope=${runtimeText(policy.scope)}`); + lines.push(renderTable([ + ["TEMP_UNSCHED", "FAILOVER", "SUCCESS", "FAILED", "MISSING", "FORWARD_200", "FORWARD_ERR", "SELECT_FAIL"], + [ + runtimeText(rules.tempUnschedulableEvents), runtimeText(failover.requests), runtimeText(failover.succeeded), + runtimeText(failover.failed), runtimeText(failover.outcomeMissing), runtimeText(forward.withHttpSuccess), + runtimeText(forward.withHttpError), runtimeText(policy.selectFailedEvents), + ], + ])); + const indexes = runtimeRecord(policy.requestIndexes) ?? {}; + const indexRows = Object.entries(indexes).flatMap(([kind, value]) => { + const item = runtimeRecord(value); + return item === null ? [] : [[kind, runtimeText(item.count), runtimeText(item.sampleRequestId), item.moreAvailable === true ? "yes" : "no"]]; + }); + if (indexRows.length > 0) lines.push("", "REQUEST INDEX", renderTable([["TYPE", "COUNT", "SAMPLE_REQUEST_ID", "MORE"], ...indexRows])); + if (typeof root.attribution === "string") lines.push("", `NOTE: ${root.attribution}`); + if (typeof policy.attribution === "string") lines.push(`NOTE: ${policy.attribution}`); + const disclosure = runtimeRecord(runtime.disclosure) ?? {}; + lines.push("", `Account: ${runtimeText(disclosure.account)}`, `Trace: ${runtimeText(disclosure.trace)}`, `Full: ${runtimeText(disclosure.complete)}`); +} + +function renderRuntimeAccountList(lines: string[], runtime: Record): void { + const group = runtimeRecord(runtime.group) ?? {}; + const accounts = runtimeRecords(runtime.accounts); + lines.push(`SUB2API RUNTIME ACCOUNTS ok=${runtime.ok === true ? "true" : "false"} group=${runtimeText(group.name)} count=${accounts.length}`); + lines.push(renderTable([ + ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "CONCURRENCY", "LOAD", "PRIORITY", "TEMP_RULES", "TEMPLATE"], + ...accounts.map((account) => { + const temp = runtimeRecord(account.tempUnschedulable) ?? {}; + return [ + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), + runtimeText(account.schedulable), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, + runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(temp.ruleCount), runtimeText(account.matchingTemplate), + ]; + }), + ])); + lines.push("", "Detail: bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account "); +} + +function renderRuntimeAccountDetail(lines: string[], runtime: Record): void { + const account = runtimeRecord(runtime.account) ?? {}; + const temp = runtimeRecord(account.tempUnschedulable) ?? {}; + lines.push(`SUB2API RUNTIME ACCOUNT ok=${runtime.ok === true ? "true" : "false"}`); + lines.push(renderTable([ + ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "PROXY", "CONCURRENCY", "LOAD", "PRIORITY", "TEMPLATE"], + [ + runtimeShort(account.accountName, 42), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), + runtimeText(account.schedulable), runtimeText(account.proxyId), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, + runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(account.matchingTemplate), + ], + ])); + lines.push(renderTable([ + ["TEMP_ENABLED", "RULES", "STATUS_CODES", "POOL_MODE"], + [runtimeText(temp.enabled), runtimeText(temp.ruleCount), runtimeText(temp.statusCodes), runtimeText(account.poolModeEnabled)], + ])); +} + +function renderRuntimeInfrastructure(lines: string[], runtime: Record): void { + const account = runtimeRecord(runtime.account) ?? {}; + const infra = runtimeRecord(runtime.infrastructure) ?? {}; + const proxy = runtimeRecord(infra.accountProxy) ?? {}; + const app = runtimeRecord(infra.appRuntime) ?? {}; + const route = runtimeRecord(infra.defaultRoute) ?? {}; + const dns = runtimeRecord(infra.dns) ?? {}; + const sockets = runtimeRecord(infra.proxySockets) ?? {}; + const listener = runtimeRecord(infra.proxyEndpointOwner) ?? {}; + const serviceLogs = runtimeRecord(infra.proxyServiceEvidence) ?? {}; + const logs = runtimeRecord(infra.recentNetworkEvidence) ?? {}; + lines.push(`SUB2API RUNTIME INFRASTRUCTURE ok=${runtime.ok === true ? "true" : "false"} since=${runtimeText(logs.window)}`); + if (runtime.ok !== true && typeof runtime.error === "string") lines.push(`ERROR: ${runtime.error}`); + lines.push(renderTable([ + ["ACCOUNT", "ID", "TYPE", "STATUS", "SCHED", "PROXY_ID", "PROXY_NAME", "PROTOCOL", "PROXY_ENDPOINT", "PROXY_STATUS"], + [ + runtimeText(account.accountName), runtimeText(account.accountId), runtimeText(account.type), runtimeText(account.status), + runtimeText(account.schedulable), runtimeText(proxy.id), runtimeText(proxy.name), runtimeText(proxy.protocol), + `${runtimeText(proxy.host)}:${runtimeText(proxy.port)}`, runtimeText(proxy.status), + ], + ])); + lines.push("", "APP RUNTIME", renderTable([ + ["MODE", "CONTAINER", "STATE", "RESTARTS", "OOM", "NETWORK_MODE", "STARTED_AT"], + [ + runtimeText(app.runtimeMode), runtimeText(app.name), runtimeText(app.state), runtimeText(app.restartCount), + runtimeText(app.oomKilled), runtimeText(app.networkMode), runtimeText(app.startedAt), + ], + ])); + const envRows = runtimeRecords(infra.proxyEnvironment); + if (envRows.length > 0) lines.push("PROXY ENVIRONMENT", renderTable([ + ["VARIABLE", "PRESENT", "SCHEME", "HOST", "PORT", "CREDENTIALS_REDACTED"], + ...envRows.map((item) => [runtimeText(item.name), runtimeText(item.present), runtimeText(item.scheme), runtimeText(item.host), runtimeText(item.port), runtimeText(item.credentialsRedacted)]), + ])); + lines.push("NETWORK", renderTable([ + ["DEFAULT_IFACE", "DEFAULT_GATEWAY", "DNS_NAMESERVERS", "DNS_SEARCH", "PROXY_RESOLVED_IPS", "SOCKETS", "ESTABLISHED"], + [ + runtimeText(route.interface), runtimeText(route.gateway), runtimeText(dns.nameservers), runtimeText(dns.search), + runtimeText(sockets.resolvedIps), runtimeText(sockets.total), runtimeText(sockets.established), + ], + ])); + const listeners = runtimeRecords(listener.listeners); + if (listeners.length > 0) lines.push("PROXY ENDPOINT OWNER", renderTable([ + ["LISTEN", "PROCESS", "PID", "UNIT", "ACTIVE", "RESTARTS", "TCP_PEERS", "UDP_PEERS"], + ...listeners.map((item) => [runtimeText(item.localEndpoint), runtimeText(item.processName), runtimeText(item.pid), runtimeText(item.systemdUnit), runtimeText(item.activeState), runtimeText(item.serviceRestarts), runtimeText(item.establishedPeers), runtimeText(item.udpPeers)]), + ])); + const serviceBuckets = runtimeRecords(serviceLogs.buckets); + if (serviceBuckets.length > 0) lines.push("PROXY SERVICE EVIDENCE", renderTable([ + ["CAUSE", "COUNT"], + ...serviceBuckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), + ])); + const serviceSamples = runtimeRecords(serviceLogs.samples); + if (serviceSamples.length > 0) lines.push("PROXY SERVICE INDEX", renderTable([ + ["AT", "CAUSE", "MESSAGE"], + ...serviceSamples.map((item) => [runtimeText(item.at), runtimeText(item.cause), runtimeShort(item.message, 100)]), + ])); + const interfaces = runtimeRecords(infra.interfaces); + if (interfaces.length > 0) lines.push("INTERFACES", renderTable([ + ["NAME", "RX_BYTES", "TX_BYTES", "RX_ERRORS", "TX_ERRORS", "RX_DROPPED", "TX_DROPPED"], + ...interfaces.map((item) => [runtimeText(item.name), runtimeText(item.rxBytes), runtimeText(item.txBytes), runtimeText(item.rxErrors), runtimeText(item.txErrors), runtimeText(item.rxDropped), runtimeText(item.txDropped)]), + ])); + const buckets = runtimeRecords(logs.buckets); + lines.push("", `RECENT NETWORK EVIDENCE matched=${runtimeText(logs.matchedEvents)} source=${runtimeText(logs.source)}`); + if (buckets.length > 0) lines.push(renderTable([ + ["CAUSE", "COUNT"], + ...buckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), + ])); + const samples = runtimeRecords(logs.samples); + if (samples.length > 0) lines.push("EVIDENCE INDEX", renderTable([ + ["AT", "REQUEST_ID", "CAUSE", "STATUS", "MESSAGE"], + ...samples.map((item) => [runtimeText(item.at), runtimeText(item.requestId), runtimeText(item.cause), runtimeText(item.statusCode), runtimeShort(item.message, 90)]), + ])); + if (typeof infra.attribution === "string") lines.push(`NOTE: ${infra.attribution}`); +} + +function renderRuntimeMutation(lines: string[], runtime: Record, options: RuntimeOptions): void { + const plans = runtimeRecords(runtime.plans); + if (plans.length > 0) { + const summary = runtimeRecord(runtime.summary) ?? {}; + lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); + lines.push(renderTable([ + ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES", "RESULT"], + ...plans.map((plan) => { + const account = runtimeRecord(plan.account) ?? {}; + const before = runtimeRecord(plan.before) ?? {}; + const after = runtimeRecord(plan.after) ?? {}; + return [ + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), + runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), runtimeText(plan.result), + ]; + }), + ])); + lines.push(`SUMMARY selected=${runtimeText(summary.selected)} changed=${runtimeText(summary.changed)} succeeded=${runtimeText(summary.succeeded)} failed=${runtimeText(summary.failed)}`); + return; + } + const plan = runtimeRecord(runtime.plan) ?? {}; + const account = runtimeRecord(plan.account) ?? runtimeRecord(runtime.account) ?? {}; + const before = runtimeRecord(plan.before) ?? {}; + const after = runtimeRecord(plan.after) ?? {}; + lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); + lines.push(renderTable([ + ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES"], + [ + runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), + runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), + ], + ])); +} diff --git a/scripts/src/platform-infra-sub2api-codex/runtime.ts b/scripts/src/platform-infra-sub2api-codex/runtime.ts index d8929ede..b04e4d96 100644 --- a/scripts/src/platform-infra-sub2api-codex/runtime.ts +++ b/scripts/src/platform-infra-sub2api-codex/runtime.ts @@ -5,7 +5,8 @@ import { readCodexPoolConfig } from "./config"; import { protectedManualAccountsForTarget } from "./public-exposure"; import { renderSub2ApiTempUnschedulableCredentials } from "./redaction"; import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote"; -import { renderedCliResult, renderTable } from "./render"; +import { compactRuntimeErrors, renderRuntimeResult } from "./runtime-render"; +import { runtimeScript } from "./runtime-remote-script"; import { parseRuntimeOptions, type RuntimeOptions } from "./runtime-options"; import { renderRuntimeMutationResult } from "./runtime-mutation-render"; import { codexPoolRuntimeTarget } from "./runtime-target"; @@ -99,2871 +100,3 @@ export async function codexPoolRuntime(config: UniDeskConfig, args: string[]): P }; return options.full || options.json ? response : renderRuntimeResult(response, options); } -function compactRuntimeErrors(value: unknown, options: RuntimeOptions): unknown { - if (options.action !== "errors" || options.full || value === null || typeof value !== "object" || Array.isArray(value)) return value; - const runtime = value as Record; - const allGroups = runtimeRecord(runtime.allGroups); - if (allGroups !== null) { - const nextPageToken = typeof allGroups.nextPageToken === "string" ? allGroups.nextPageToken : null; - const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; - const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; - return { - ok: runtime.ok, - operation: runtime.operation, - mutation: runtime.mutation, - endpoint: runtime.endpoint, - allGroups, - disclosure: { - group: `${baseCommand} --group ${platformFlag}`, - nextPage: nextPageToken === null ? null : `${baseCommand} --all-groups${platformFlag} --page-token ${nextPageToken}`, - note: "The overview uses stable group-id pagination. Use group drilldown for account root causes and request indexes.", - }, - valuesPrinted: false, - }; - } - const errors = runtimeRecord(runtime.errors); - if (errors === null) return value; - const effectiveness = runtimeRecord(errors.policyEffectiveness) ?? {}; - const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; - const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; - const availabilityGroup = runtimeRecord(availability.group) ?? {}; - const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; - const concurrencyGroup = runtimeRecord(concurrency.group) ?? {}; - const sink = runtimeRecord(nativeOps.systemLogSink) ?? {}; - const sourceStatus = runtimeRecord(effectiveness.sourceStatus) ?? {}; - const drilldown = runtimeRecord(effectiveness.requestDrilldown) ?? {}; - const requestIndexes = Object.fromEntries(Object.entries(drilldown).flatMap(([key, item]) => { - if (!Array.isArray(item)) return []; - const ids = item.filter((entry): entry is string => typeof entry === "string"); - if (ids.length === 0) return []; - return [[key, { - count: ids.length, - sampleRequestId: ids[0], - moreAvailable: ids.length > 1, - }]]; - })); - const baseCommand = `bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors --target ${options.targetId} --since ${options.since}`; - const groupFlag = options.group === null ? "" : ` --group ${options.group}`; - const platformFlag = options.platform === null ? "" : ` --platform ${options.platform}`; - if (options.json && options.account === null) { - return { - ok: runtime.ok, - operation: runtime.operation, - mutation: runtime.mutation, - group: runtime.group, - endpoint: runtime.endpoint, - errors: { - window: errors.window, - nativeOps: { - source: nativeOps.source, - overview: nativeOps.overview, - accountQuality: compactAccountQuality(nativeOps.accountQuality, false), - accountAvailability: { - status: availability.status, - totalAccounts: availabilityGroup.total_accounts, - availableCount: availabilityGroup.available_count, - rateLimitCount: availabilityGroup.rate_limit_count, - errorCount: availabilityGroup.error_count, - unavailableAccounts: Array.isArray(availability.unavailableAccounts) - ? availability.unavailableAccounts.map((item) => { - const account = runtimeRecord(item) ?? {}; - return { - accountId: account.accountId, - accountName: account.accountName, - status: account.status, - reason: account.error, - }; - }) - : [], - }, - concurrency: { - status: concurrency.status, - currentInUse: concurrencyGroup.current_in_use, - maxCapacity: concurrencyGroup.max_capacity, - loadPercentage: concurrencyGroup.load_percentage, - waitingInQueue: concurrencyGroup.waiting_in_queue, - }, - }, - }, - disclosure: { - account: `${baseCommand}${groupFlag}${platformFlag} --account `, - trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", - note: "This group overview keeps every account name and quality summary. Use account drilldown for request indexes and complete failure evidence.", - }, - valuesPrinted: false, - }; - } - return { - ok: runtime.ok, - operation: runtime.operation, - mutation: runtime.mutation, - group: runtime.group, - endpoint: runtime.endpoint, - errors: { - window: errors.window, - nativeOps: { - source: nativeOps.source, - overview: nativeOps.overview, - responseHeaderTimeout: options.account === null || !options.json ? nativeOps.responseHeaderTimeout : undefined, - requestTiming: options.account === null || !options.json ? compactRequestTiming(nativeOps.requestTiming) : undefined, - customerErrors: compactCustomerErrors(nativeOps.customerErrors, options.account !== null || !options.json), - accountQuality: compactAccountQuality(nativeOps.accountQuality, options.account !== null || !options.json), - accountAvailability: { - status: availability.status, - totalAccounts: availabilityGroup.total_accounts, - availableCount: availabilityGroup.available_count, - rateLimitCount: availabilityGroup.rate_limit_count, - errorCount: availabilityGroup.error_count, - unavailableAccounts: Array.isArray(availability.unavailableAccounts) - ? availability.unavailableAccounts.map((item) => { - const account = runtimeRecord(item) ?? {}; - return { - accountId: account.accountId, - accountName: account.accountName, - status: account.status, - reason: account.error, - }; - }) - : [], - }, - concurrency: { - status: concurrency.status, - currentInUse: concurrencyGroup.current_in_use, - maxCapacity: concurrencyGroup.max_capacity, - loadPercentage: concurrencyGroup.load_percentage, - waitingInQueue: concurrencyGroup.waiting_in_queue, - }, - systemLogSink: { - status: sink.status, - queueDepth: sink.queueDepth, - droppedCount: sink.droppedCount, - writeFailedCount: sink.writeFailedCount, - }, - }, - rootCauseAnalysis: errors.rootCauseAnalysis, - policyEffectiveness: { - source: effectiveness.source, - scope: effectiveness.scope, - attribution: effectiveness.attribution, - assessment: effectiveness.assessment, - ruleMatches: effectiveness.ruleMatches, - failover: effectiveness.failover, - forwardFailure: effectiveness.forwardFailure, - selectFailedEvents: effectiveness.selectFailedEvents, - requestIndexes, - sourceStatus: { - status: sourceStatus.status, - fallbackUsed: sourceStatus.fallbackUsed, - fallbackReason: sourceStatus.fallbackReason, - }, - }, - }, - disclosure: { - complete: `${baseCommand}${groupFlag}${platformFlag}${options.account === null ? "" : " --account "} --full`, - account: `${baseCommand}${groupFlag}${platformFlag} --account `, - trace: "bun scripts/cli.ts platform-infra sub2api codex-pool trace --target --request-id ", - note: "Use requestIndexes for trace drilldown and --full for the complete finite index.", - }, - valuesPrinted: false, - }; -} -function compactRequestTiming(value: unknown): unknown { - const timing = runtimeRecord(value); - if (timing === null) return value; - return { - status: timing.status, - reason: timing.reason, - source: timing.source, - ttft: timing.ttft, - duration: timing.duration, - observedFloorSeconds: timing.observedFloorSeconds, - attribution: timing.attribution, - }; -} -function compactCustomerErrors(value: unknown, detailed: boolean): unknown { - const profile = runtimeRecord(value); - if (profile === null) return value; - return { - status: profile.status, - reason: profile.reason, - source: profile.source, - total: profile.total, - modelBuckets: runtimeRecords(profile.modelBuckets).slice(0, 5), - accountBuckets: detailed ? runtimeRecords(profile.accountBuckets).slice(0, 5) : undefined, - statusBuckets: runtimeRecords(profile.statusBuckets).slice(0, 5), - streamBuckets: runtimeRecords(profile.streamBuckets).slice(0, 5), - messageBuckets: runtimeRecords(profile.messageBuckets).slice(0, detailed ? 10 : 5), - }; -} -function compactAccountQuality(value: unknown, detailed: boolean): unknown { - const quality = runtimeRecord(value); - if (quality === null) return value; - return { - source: quality.source, - scope: quality.scope, - window: quality.window, - scoreRule: (() => { - const rule = runtimeRecord(quality.scoreRule) ?? {}; - return detailed ? { - reliabilityWeight: rule.reliabilityWeight, - latencyWeight: rule.latencyWeight, - availabilityWeight: rule.availabilityWeight, - failureDefinition: rule.failureDefinition, - missingMetric: rule.missingMetric, - } : { - reliabilityWeight: rule.reliabilityWeight, - latencyWeight: rule.latencyWeight, - availabilityWeight: rule.availabilityWeight, - }; - })(), - accounts: runtimeRecords(quality.accounts).map((account) => detailed ? { - accountId: account.accountId, - accountName: account.accountName, - status: account.status, - currentlyAvailable: account.currentlyAvailable, - score: account.score, - grade: account.grade, - assessment: account.assessment, - confidence: account.confidence, - observedAttempts: account.observedAttempts, - successRequests: account.successRequests, - failureRequests: account.failureRequests, - failureRate: account.failureRate, - firstTokenSamples: account.firstTokenSamples, - firstTokenCoverage: account.firstTokenCoverage, - ttftP95Ms: account.ttftP95Ms, - customerErrorRequests: account.customerErrorRequests, - failoverRequests: account.failoverRequests, - failoverRecovered: account.failoverRecovered, - failoverFailed: account.failoverFailed, - sameAccountRetryEvents: account.sameAccountRetryEvents, - tempUnschedulableEvents: account.tempUnschedulableEvents, - forwardFailedRequests: account.forwardFailedRequests, - reasons: account.reasons, - } : { - accountId: account.accountId, - accountName: account.accountName, - currentlyAvailable: account.currentlyAvailable, - score: account.score, - grade: account.grade, - confidence: account.confidence, - observedAttempts: account.observedAttempts, - failureRate: account.failureRate, - ttftP95Ms: account.ttftP95Ms, - customerErrorRequests: account.customerErrorRequests, - failoverRequests: account.failoverRequests, - failoverRecovered: account.failoverRecovered, - failoverFailed: account.failoverFailed, - }), - attribution: detailed ? quality.attribution : undefined, - }; -} -function runtimeRecord(value: unknown): Record | null { - return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : null; -} - -function runtimeRecords(value: unknown): Record[] { - return Array.isArray(value) ? value.flatMap((item) => { - const record = runtimeRecord(item); - return record === null ? [] : [record]; - }) : []; -} -function runtimeText(value: unknown): string { - if (value === null || value === undefined || value === "") return "-"; - if (typeof value === "boolean") return value ? "true" : "false"; - if (Array.isArray(value)) return value.map(runtimeText).join(","); - return String(value).replace(/\s+/gu, " ").trim(); -} - -function runtimeShort(value: unknown, max = 42): string { - const text = runtimeText(value); - return text.length <= max ? text : `${text.slice(0, max - 1)}…`; -} - -function runtimeRate(value: unknown): string { - return typeof value === "number" && Number.isFinite(value) ? `${(value * 100).toFixed(2)}%` : "-"; -} - -function renderRuntimeResult(response: Record, options: RuntimeOptions): RenderedCliResult { - const runtime = runtimeRecord(response.runtime) ?? {}; - const ok = response.ok === true; - const lines: string[] = []; - if (options.action === "errors") renderRuntimeErrors(lines, runtime, options); - else if (options.action === "infrastructure") renderRuntimeInfrastructure(lines, runtime); - else if (options.action === "list") renderRuntimeAccountList(lines, runtime); - else if (options.action === "get") renderRuntimeAccountDetail(lines, runtime); - else renderRuntimeMutation(lines, runtime, options); - lines.push("", `JSON: bun scripts/cli.ts platform-infra sub2api codex-pool runtime ${options.action} --json`); - return renderedCliResult(ok, `platform-infra sub2api codex-pool runtime ${options.action}`, lines.join("\n")); -} - -function renderRuntimeErrors(lines: string[], runtime: Record, options: RuntimeOptions): void { - const allGroups = runtimeRecord(runtime.allGroups); - if (allGroups !== null) { - const totals = runtimeRecord(allGroups.pageTotals) ?? {}; - const timeout = runtimeRecord(allGroups.responseHeaderTimeout) ?? {}; - const timing = runtimeRecord(allGroups.requestTiming) ?? {}; - const groups = runtimeRecords(allGroups.groups); - lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=all-groups since=${options.since}`); - lines.push(renderTable([ - ["GROUPS", "RETURNED", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UPSTREAM_ERR", "UP_RATE", "ATTENTION"], - [ - runtimeText(allGroups.totalGroups), runtimeText(allGroups.returnedGroups), runtimeText(totals.requestCount), - runtimeText(totals.errorCount), runtimeRate(totals.errorRate), runtimeText(totals.upstreamErrorCount), - runtimeRate(totals.upstreamErrorRate), runtimeText(totals.groupsNeedingAttention), - ], - ])); - lines.push(renderTable([ - ["OPENAI_HEADER_TIMEOUT", "SOURCE", "GENERIC_TIMEOUT", "GENERIC_APPLIES_TO_OPENAI", "PAGE_MAX_TTFT_P99_MS", "OBSERVED_FLOOR_SECONDS"], - [ - timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source), - `${runtimeText(timeout.genericResponseHeaderTimeoutSeconds)}s`, runtimeText(timeout.genericAppliesToOpenAI), - runtimeText(timing.maxTtftP99Ms), runtimeText(timing.observedFloorSeconds), - ], - ])); - if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); - lines.push("", "GROUPS"); - lines.push(renderTable([ - ["NAME", "ID", "PLATFORM", "STATUS", "REQ", "CLIENT_ERR", "ERR_RATE", "UP_ERR", "UP_RATE", "TTFT_P99_MS", "DUR_P99_MS", "ACCOUNTS", "AVAILABLE", "USE/CAP", "QUEUE", "ATTN"], - ...groups.map((group) => [ - runtimeShort(group.groupName, 28), runtimeText(group.groupId), runtimeText(group.platform), runtimeText(group.status), - runtimeText(group.requestCount), runtimeText(group.errorCount), runtimeRate(group.errorRate), runtimeText(group.upstreamErrorCount), - runtimeRate(group.upstreamErrorRate), runtimeText(group.ttftP99Ms), runtimeText(group.durationP99Ms), runtimeText(group.totalAccounts), runtimeText(group.availableCount), - `${runtimeText(group.currentInUse)}/${runtimeText(group.maxCapacity)}`, runtimeText(group.waitingInQueue), group.needsAttention === true ? "yes" : "no", - ]), - ])); - const disclosure = runtimeRecord(runtime.disclosure) ?? {}; - lines.push("", `Detail: ${runtimeText(disclosure.group)}`); - if (typeof disclosure.nextPage === "string") lines.push(`Next: ${disclosure.nextPage}`); - return; - } - const group = runtimeRecord(runtime.group) ?? {}; - const errors = runtimeRecord(runtime.errors) ?? {}; - const nativeOps = runtimeRecord(errors.nativeOps) ?? {}; - const overview = runtimeRecord(nativeOps.overview) ?? {}; - const availability = runtimeRecord(nativeOps.accountAvailability) ?? {}; - const concurrency = runtimeRecord(nativeOps.concurrency) ?? {}; - const timeout = runtimeRecord(nativeOps.responseHeaderTimeout) ?? {}; - const timing = runtimeRecord(nativeOps.requestTiming) ?? {}; - const ttft = runtimeRecord(timing.ttft) ?? {}; - const duration = runtimeRecord(timing.duration) ?? {}; - const customerErrors = runtimeRecord(nativeOps.customerErrors) ?? {}; - const accountQuality = runtimeRecord(nativeOps.accountQuality) ?? {}; - lines.push(`SUB2API RUNTIME ERRORS ok=${runtime.ok === true ? "true" : "false"} scope=group since=${options.since}`); - lines.push(renderTable([ - ["GROUP", "ID", "PLATFORM", "REQUESTS", "CLIENT_ERR", "ERR_RATE", "UP_RATE", "BUSINESS_LIMIT", "HEALTH"], - [ - runtimeShort(group.name, 30), runtimeText(group.id), runtimeText(group.platform), runtimeText(overview.requestCount), - runtimeText(overview.errorCount), runtimeRate(overview.errorRate), runtimeRate(overview.upstreamErrorRate), - runtimeText(overview.businessLimitedCount), runtimeText(overview.healthScore), - ], - ])); - lines.push(renderTable([ - ["OPENAI_HEADER_TIMEOUT", "SOURCE", "TTFT_P50_MS", "TTFT_P95_MS", "TTFT_P99_MS", "TTFT_MAX_MS", "DURATION_P99_MS", "DURATION_MAX_MS"], - [ - timeout.disabled === true ? "disabled" : `${runtimeText(timeout.effectiveSeconds)}s`, runtimeText(timeout.source), - runtimeText(ttft.p50_ms), runtimeText(ttft.p95_ms), runtimeText(ttft.p99_ms), runtimeText(ttft.max_ms), - runtimeText(duration.p99_ms), runtimeText(duration.max_ms), - ], - ])); - const candidates = runtimeRecords(timing.candidates); - if (candidates.length > 0) lines.push("", "TIMEOUT CANDIDATES", renderTable([ - ["SECONDS", "OBSERVED_FLOOR", "TTFT_P99_HEADROOM_MS", "TTFT_P99_BELOW", "ALL_DUR>=", "ERROR_DUR>=", "ASSESSMENT"], - ...candidates.map((candidate) => [ - runtimeText(candidate.seconds), runtimeText(candidate.observedFloor), runtimeText(candidate.ttftP99HeadroomMs), runtimeText(candidate.ttftP99Below), - runtimeText(candidate.totalDurationAtLeastCount), runtimeText(candidate.errorDurationAtLeastCount), runtimeText(candidate.assessment), - ]), - ])); - if (typeof timing.attribution === "string") lines.push(`NOTE: ${timing.attribution}`); - const modelTiming = runtimeRecords(timing.modelTiming); - if (modelTiming.length > 0) lines.push("", "MODEL TIMING", renderTable([ - ["MODEL", "REQUESTS", "FIRST_TOKEN_ROWS", "COVERAGE", "AVG_TTFT_MS", "AVG_DURATION_MS", "AVG_TOKENS_S"], - ...modelTiming.map((item) => [runtimeShort(item.model, 32), runtimeText(item.requestCount), runtimeText(item.requestsWithFirstToken), runtimeRate(item.firstTokenCoverage), runtimeText(item.avgFirstTokenMs), runtimeText(item.avgDurationMs), runtimeText(item.avgTokensPerSec)]), - ])); - const qualityAccounts = runtimeRecords(accountQuality.accounts); - if (qualityAccounts.length > 0) { - lines.push("", "ACCOUNT QUALITY", renderTable([ - ["ACCOUNT", "ID", "SCORE", "GRADE", "QUALITY", "CONF", "ATTEMPTS", "SUCCESS", "FAIL", "FAIL_RATE", "TTFT_P95_MS", "TTFT_N", "AVAILABLE", "REASONS"], - ...qualityAccounts.map((item) => [ - runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.score), runtimeText(item.grade), runtimeText(item.assessment), runtimeText(item.confidence), - runtimeText(item.observedAttempts), runtimeText(item.successRequests), runtimeText(item.failureRequests), runtimeRate(item.failureRate), - runtimeText(item.ttftP95Ms), runtimeText(item.firstTokenSamples), runtimeText(item.currentlyAvailable), - runtimeShort(Array.isArray(item.reasons) ? item.reasons.join(",") : "", 58), - ]), - ])); - const controlRows = qualityAccounts.filter((item) => [ - item.failoverRequests, item.sameAccountRetryEvents, item.tempUnschedulableEvents, item.forwardFailedRequests, item.customerErrorRequests, - ].some((value) => typeof value === "number" && value > 0)); - if (controlRows.length > 0) lines.push("ACCOUNT FAILURE CONTROL", renderTable([ - ["ACCOUNT", "ID", "FAILOVER", "RECOVERED", "FAILED", "MISSING", "RETRY", "TEMP_UNSCHED", "FORWARD_FAIL", "CUSTOMER_ERR"], - ...controlRows.map((item) => [ - runtimeText(item.accountName), runtimeText(item.accountId), runtimeText(item.failoverRequests), runtimeText(item.failoverRecovered), - runtimeText(item.failoverFailed), runtimeText(item.failoverOutcomeMissing), runtimeText(item.sameAccountRetryEvents), - runtimeText(item.tempUnschedulableEvents), runtimeText(item.forwardFailedRequests), runtimeText(item.customerErrorRequests), - ]), - ])); - const scoreRule = runtimeRecord(accountQuality.scoreRule) ?? {}; - lines.push(`SCORE RULE reliability=${runtimeText(scoreRule.reliabilityWeight)} latency=${runtimeText(scoreRule.latencyWeight)} availability=${runtimeText(scoreRule.availabilityWeight)} confidence=separate`); - if (typeof accountQuality.attribution === "string") lines.push(`NOTE: ${accountQuality.attribution}`); - } - const errorModels = runtimeRecords(customerErrors.modelBuckets); - const errorAccounts = runtimeRecords(customerErrors.accountBuckets); - const errorDimensions = [ - ...runtimeRecords(customerErrors.statusBuckets).map((item) => ["status", runtimeText(item.value), runtimeText(item.count)]), - ...runtimeRecords(customerErrors.streamBuckets).map((item) => ["mode", runtimeText(item.value), runtimeText(item.count)]), - ...runtimeRecords(customerErrors.phaseBuckets).map((item) => ["phase", runtimeText(item.value), runtimeText(item.count)]), - ...runtimeRecords(customerErrors.endpointBuckets).map((item) => ["endpoint", runtimeText(item.value), runtimeText(item.count)]), - ...runtimeRecords(customerErrors.pathBuckets).map((item) => ["path", runtimeText(item.value), runtimeText(item.count)]), - ]; - if (errorModels.length > 0) lines.push("", `CUSTOMER ERRORS source=${runtimeText(customerErrors.source)} total=${runtimeText(customerErrors.total)}`, renderTable([ - ["MODEL", "COUNT"], ...errorModels.map((item) => [runtimeShort(item.value, 40), runtimeText(item.count)]), - ])); - if (errorAccounts.length > 0) lines.push(renderTable([ - ["ACCOUNT_ID", "COUNT"], ...errorAccounts.map((item) => [runtimeText(item.value), runtimeText(item.count)]), - ])); - if (errorDimensions.length > 0) lines.push(renderTable([["DIMENSION", "VALUE", "COUNT"], ...errorDimensions])); - const errorSamples = runtimeRecords(customerErrors.samples); - if (errorSamples.length > 0) lines.push("CUSTOMER ERROR INDEX", renderTable([ - ["REQUEST_ID", "MODEL", "STATUS", "ACCOUNT", "MODE", "PATH", "MESSAGE"], - ...errorSamples.map((item) => [runtimeText(item.requestId), runtimeShort(item.model, 26), runtimeText(item.statusCode), runtimeText(item.accountId), runtimeText(item.mode), runtimeShort(item.path, 34), runtimeShort(item.message, 48)]), - ])); - lines.push(renderTable([ - ["ACCOUNTS", "AVAILABLE", "RATE_LIMIT", "ERROR", "IN_USE", "CAPACITY", "QUEUE"], - [ - runtimeText(availability.totalAccounts), runtimeText(availability.availableCount), runtimeText(availability.rateLimitCount), - runtimeText(availability.errorCount), runtimeText(concurrency.currentInUse), runtimeText(concurrency.maxCapacity), runtimeText(concurrency.waitingInQueue), - ], - ])); - const unavailable = runtimeRecords(availability.unavailableAccounts); - if (unavailable.length > 0) { - lines.push("", "UNAVAILABLE ACCOUNTS", renderTable([ - ["ACCOUNT", "ID", "STATUS", "REASON"], - ...unavailable.map((account) => [runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.status), runtimeShort(account.reason, 70)]), - ])); - } - const root = runtimeRecord(errors.rootCauseAnalysis) ?? {}; - const causes = runtimeRecords(root.causeBuckets); - if (causes.length > 0) { - lines.push("", `ROOT CAUSES scope=${runtimeText(root.scope)}`, renderTable([ - ["CAUSE", "COUNT"], - ...causes.map((cause) => [runtimeText(cause.cause), runtimeText(cause.count)]), - ])); - } - const policy = runtimeRecord(errors.policyEffectiveness) ?? {}; - const rules = runtimeRecord(policy.ruleMatches) ?? {}; - const failover = runtimeRecord(policy.failover) ?? {}; - const forward = runtimeRecord(policy.forwardFailure) ?? {}; - lines.push("", `POLICY EFFECTIVENESS scope=${runtimeText(policy.scope)}`); - lines.push(renderTable([ - ["TEMP_UNSCHED", "FAILOVER", "SUCCESS", "FAILED", "MISSING", "FORWARD_200", "FORWARD_ERR", "SELECT_FAIL"], - [ - runtimeText(rules.tempUnschedulableEvents), runtimeText(failover.requests), runtimeText(failover.succeeded), - runtimeText(failover.failed), runtimeText(failover.outcomeMissing), runtimeText(forward.withHttpSuccess), - runtimeText(forward.withHttpError), runtimeText(policy.selectFailedEvents), - ], - ])); - const indexes = runtimeRecord(policy.requestIndexes) ?? {}; - const indexRows = Object.entries(indexes).flatMap(([kind, value]) => { - const item = runtimeRecord(value); - return item === null ? [] : [[kind, runtimeText(item.count), runtimeText(item.sampleRequestId), item.moreAvailable === true ? "yes" : "no"]]; - }); - if (indexRows.length > 0) lines.push("", "REQUEST INDEX", renderTable([["TYPE", "COUNT", "SAMPLE_REQUEST_ID", "MORE"], ...indexRows])); - if (typeof root.attribution === "string") lines.push("", `NOTE: ${root.attribution}`); - if (typeof policy.attribution === "string") lines.push(`NOTE: ${policy.attribution}`); - const disclosure = runtimeRecord(runtime.disclosure) ?? {}; - lines.push("", `Account: ${runtimeText(disclosure.account)}`, `Trace: ${runtimeText(disclosure.trace)}`, `Full: ${runtimeText(disclosure.complete)}`); -} - -function renderRuntimeAccountList(lines: string[], runtime: Record): void { - const group = runtimeRecord(runtime.group) ?? {}; - const accounts = runtimeRecords(runtime.accounts); - lines.push(`SUB2API RUNTIME ACCOUNTS ok=${runtime.ok === true ? "true" : "false"} group=${runtimeText(group.name)} count=${accounts.length}`); - lines.push(renderTable([ - ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "CONCURRENCY", "LOAD", "PRIORITY", "TEMP_RULES", "TEMPLATE"], - ...accounts.map((account) => { - const temp = runtimeRecord(account.tempUnschedulable) ?? {}; - return [ - runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), - runtimeText(account.schedulable), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, - runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(temp.ruleCount), runtimeText(account.matchingTemplate), - ]; - }), - ])); - lines.push("", "Detail: bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account "); -} - -function renderRuntimeAccountDetail(lines: string[], runtime: Record): void { - const account = runtimeRecord(runtime.account) ?? {}; - const temp = runtimeRecord(account.tempUnschedulable) ?? {}; - lines.push(`SUB2API RUNTIME ACCOUNT ok=${runtime.ok === true ? "true" : "false"}`); - lines.push(renderTable([ - ["NAME", "ID", "MANAGEMENT", "STATUS", "SCHED", "PROXY", "CONCURRENCY", "LOAD", "PRIORITY", "TEMPLATE"], - [ - runtimeShort(account.accountName, 42), runtimeText(account.accountId), runtimeText(account.management), runtimeText(account.status), - runtimeText(account.schedulable), runtimeText(account.proxyId), `${runtimeText(account.currentConcurrency)}/${runtimeText(account.concurrencyLimit)}`, - runtimeText(account.loadFactor), runtimeText(account.priority), runtimeText(account.matchingTemplate), - ], - ])); - lines.push(renderTable([ - ["TEMP_ENABLED", "RULES", "STATUS_CODES", "POOL_MODE"], - [runtimeText(temp.enabled), runtimeText(temp.ruleCount), runtimeText(temp.statusCodes), runtimeText(account.poolModeEnabled)], - ])); -} - -function renderRuntimeInfrastructure(lines: string[], runtime: Record): void { - const account = runtimeRecord(runtime.account) ?? {}; - const infra = runtimeRecord(runtime.infrastructure) ?? {}; - const proxy = runtimeRecord(infra.accountProxy) ?? {}; - const app = runtimeRecord(infra.appRuntime) ?? {}; - const route = runtimeRecord(infra.defaultRoute) ?? {}; - const dns = runtimeRecord(infra.dns) ?? {}; - const sockets = runtimeRecord(infra.proxySockets) ?? {}; - const listener = runtimeRecord(infra.proxyEndpointOwner) ?? {}; - const serviceLogs = runtimeRecord(infra.proxyServiceEvidence) ?? {}; - const logs = runtimeRecord(infra.recentNetworkEvidence) ?? {}; - lines.push(`SUB2API RUNTIME INFRASTRUCTURE ok=${runtime.ok === true ? "true" : "false"} since=${runtimeText(logs.window)}`); - if (runtime.ok !== true && typeof runtime.error === "string") lines.push(`ERROR: ${runtime.error}`); - lines.push(renderTable([ - ["ACCOUNT", "ID", "TYPE", "STATUS", "SCHED", "PROXY_ID", "PROXY_NAME", "PROTOCOL", "PROXY_ENDPOINT", "PROXY_STATUS"], - [ - runtimeText(account.accountName), runtimeText(account.accountId), runtimeText(account.type), runtimeText(account.status), - runtimeText(account.schedulable), runtimeText(proxy.id), runtimeText(proxy.name), runtimeText(proxy.protocol), - `${runtimeText(proxy.host)}:${runtimeText(proxy.port)}`, runtimeText(proxy.status), - ], - ])); - lines.push("", "APP RUNTIME", renderTable([ - ["MODE", "CONTAINER", "STATE", "RESTARTS", "OOM", "NETWORK_MODE", "STARTED_AT"], - [ - runtimeText(app.runtimeMode), runtimeText(app.name), runtimeText(app.state), runtimeText(app.restartCount), - runtimeText(app.oomKilled), runtimeText(app.networkMode), runtimeText(app.startedAt), - ], - ])); - const envRows = runtimeRecords(infra.proxyEnvironment); - if (envRows.length > 0) lines.push("PROXY ENVIRONMENT", renderTable([ - ["VARIABLE", "PRESENT", "SCHEME", "HOST", "PORT", "CREDENTIALS_REDACTED"], - ...envRows.map((item) => [runtimeText(item.name), runtimeText(item.present), runtimeText(item.scheme), runtimeText(item.host), runtimeText(item.port), runtimeText(item.credentialsRedacted)]), - ])); - lines.push("NETWORK", renderTable([ - ["DEFAULT_IFACE", "DEFAULT_GATEWAY", "DNS_NAMESERVERS", "DNS_SEARCH", "PROXY_RESOLVED_IPS", "SOCKETS", "ESTABLISHED"], - [ - runtimeText(route.interface), runtimeText(route.gateway), runtimeText(dns.nameservers), runtimeText(dns.search), - runtimeText(sockets.resolvedIps), runtimeText(sockets.total), runtimeText(sockets.established), - ], - ])); - const listeners = runtimeRecords(listener.listeners); - if (listeners.length > 0) lines.push("PROXY ENDPOINT OWNER", renderTable([ - ["LISTEN", "PROCESS", "PID", "UNIT", "ACTIVE", "RESTARTS", "TCP_PEERS", "UDP_PEERS"], - ...listeners.map((item) => [runtimeText(item.localEndpoint), runtimeText(item.processName), runtimeText(item.pid), runtimeText(item.systemdUnit), runtimeText(item.activeState), runtimeText(item.serviceRestarts), runtimeText(item.establishedPeers), runtimeText(item.udpPeers)]), - ])); - const serviceBuckets = runtimeRecords(serviceLogs.buckets); - if (serviceBuckets.length > 0) lines.push("PROXY SERVICE EVIDENCE", renderTable([ - ["CAUSE", "COUNT"], - ...serviceBuckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), - ])); - const serviceSamples = runtimeRecords(serviceLogs.samples); - if (serviceSamples.length > 0) lines.push("PROXY SERVICE INDEX", renderTable([ - ["AT", "CAUSE", "MESSAGE"], - ...serviceSamples.map((item) => [runtimeText(item.at), runtimeText(item.cause), runtimeShort(item.message, 100)]), - ])); - const interfaces = runtimeRecords(infra.interfaces); - if (interfaces.length > 0) lines.push("INTERFACES", renderTable([ - ["NAME", "RX_BYTES", "TX_BYTES", "RX_ERRORS", "TX_ERRORS", "RX_DROPPED", "TX_DROPPED"], - ...interfaces.map((item) => [runtimeText(item.name), runtimeText(item.rxBytes), runtimeText(item.txBytes), runtimeText(item.rxErrors), runtimeText(item.txErrors), runtimeText(item.rxDropped), runtimeText(item.txDropped)]), - ])); - const buckets = runtimeRecords(logs.buckets); - lines.push("", `RECENT NETWORK EVIDENCE matched=${runtimeText(logs.matchedEvents)} source=${runtimeText(logs.source)}`); - if (buckets.length > 0) lines.push(renderTable([ - ["CAUSE", "COUNT"], - ...buckets.map((item) => [runtimeText(item.cause), runtimeText(item.count)]), - ])); - const samples = runtimeRecords(logs.samples); - if (samples.length > 0) lines.push("EVIDENCE INDEX", renderTable([ - ["AT", "REQUEST_ID", "CAUSE", "STATUS", "MESSAGE"], - ...samples.map((item) => [runtimeText(item.at), runtimeText(item.requestId), runtimeText(item.cause), runtimeText(item.statusCode), runtimeShort(item.message, 90)]), - ])); - if (typeof infra.attribution === "string") lines.push(`NOTE: ${infra.attribution}`); -} - -function renderRuntimeMutation(lines: string[], runtime: Record, options: RuntimeOptions): void { - const plans = runtimeRecords(runtime.plans); - if (plans.length > 0) { - const summary = runtimeRecord(runtime.summary) ?? {}; - lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); - lines.push(renderTable([ - ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES", "RESULT"], - ...plans.map((plan) => { - const account = runtimeRecord(plan.account) ?? {}; - const before = runtimeRecord(plan.before) ?? {}; - const after = runtimeRecord(plan.after) ?? {}; - return [ - runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), - runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), runtimeText(plan.result), - ]; - }), - ])); - lines.push(`SUMMARY selected=${runtimeText(summary.selected)} changed=${runtimeText(summary.changed)} succeeded=${runtimeText(summary.succeeded)} failed=${runtimeText(summary.failed)}`); - return; - } - const plan = runtimeRecord(runtime.plan) ?? {}; - const account = runtimeRecord(plan.account) ?? runtimeRecord(runtime.account) ?? {}; - const before = runtimeRecord(plan.before) ?? {}; - const after = runtimeRecord(plan.after) ?? {}; - lines.push(`SUB2API RUNTIME ${options.action.toUpperCase()} ok=${runtime.ok === true ? "true" : "false"} mode=${runtimeText(runtime.mode)} mutation=${runtimeText(runtime.mutation)}`); - lines.push(renderTable([ - ["ACCOUNT", "ID", "CHANGE", "TEMPLATE", "BEFORE_RULES", "AFTER_RULES", "BEFORE_CODES", "AFTER_CODES"], - [ - runtimeShort(account.accountName, 36), runtimeText(account.accountId), runtimeText(plan.change), runtimeText(plan.template), - runtimeText(before.ruleCount), runtimeText(after.ruleCount), runtimeText(before.statusCodes), runtimeText(after.statusCodes), - ], - ])); -} - -function pyJson(value: unknown): string { - return `json.loads(${JSON.stringify(JSON.stringify(value))})`; -} - -function runtimeScript(payload: unknown, target: ReturnType): string { - return ` -set -u -python3 - <<'PY' -import json -import re -import socket -import struct -import subprocess -import sys -from datetime import datetime, timedelta, timezone -from urllib.parse import quote, urlsplit - -RUNTIME_MODE = ${pyJson(target.runtimeMode)} -NAMESPACE = ${pyJson(target.namespace)} -HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)} -HOST_DOCKER_ENV_PATH = ${pyJson(target.hostDockerEnvPath)} -HOST_DOCKER_APP_CONTAINER = "sub2api-app" -APP_SECRET_NAME = ${pyJson(target.appSecretName)} -PAYLOAD = ${pyJson(payload)} - -def run(cmd, input_bytes=None): - return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - -def text(value, limit=1000): - if isinstance(value, bytes): - value = value.decode("utf-8", errors="replace") - return str(value)[-limit:] - -def docker(args): - proc = run(["docker", *args]) - if proc.returncode == 0: - return proc - sudo_proc = run(["sudo", "-n", "docker", *args]) - return sudo_proc if sudo_proc.returncode == 0 else proc - -def kubectl(args, input_bytes=None): - return run(["kubectl", *args], input_bytes) - -def kube_json(args, label): - proc = kubectl([*args, "-o", "json"]) - if proc.returncode != 0: - raise RuntimeError(f"{label} failed: {text(proc.stderr)}") - return json.loads(proc.stdout.decode("utf-8")) - -def read_host_env(): - try: - with open(HOST_DOCKER_ENV_PATH, "r", encoding="utf-8") as handle: - lines = handle.read().splitlines() - except PermissionError: - proc = run(["sudo", "-n", "cat", HOST_DOCKER_ENV_PATH]) - if proc.returncode != 0: - raise RuntimeError("read host env failed: " + text(proc.stderr)) - lines = proc.stdout.decode("utf-8", errors="replace").splitlines() - values = {} - for line in lines: - stripped = line.strip() - if not stripped or stripped.startswith("#") or "=" not in stripped: - continue - key, value = stripped.split("=", 1) - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): - value = value[1:-1] - values[key.strip()] = value - return values - -def select_app_pod(): - if RUNTIME_MODE == "host-docker": - return HOST_DOCKER_APP_CONTAINER - pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api"], "sub2api pods").get("items") or [] - running = [item for item in pods if (item.get("status") or {}).get("phase") == "Running"] - if not running: - raise RuntimeError("sub2api app pod not found") - return running[0]["metadata"]["name"] - -APP_POD = select_app_pod() - -def config_value(name, key, fallback=None): - if RUNTIME_MODE == "host-docker": - return read_host_env().get(key) or fallback - data = kube_json(["-n", NAMESPACE, "get", "configmap", name], f"configmap/{name}").get("data") or {} - return data.get(key) or fallback - -def secret_value(name, key): - if RUNTIME_MODE == "host-docker": - return read_host_env().get(key) - import base64 - data = kube_json(["-n", NAMESPACE, "get", "secret", name], f"secret/{name}").get("data") or {} - return base64.b64decode(data[key]).decode("utf-8") if key in data else None - -def parse_curl(proc): - output = proc.stdout.decode("utf-8", errors="replace") - marker = "\\n__HTTP_CODE__:" - pos = output.rfind(marker) - if pos < 0: - return {"ok": False, "httpStatus": 0, "json": None, "body": output, "stderr": text(proc.stderr)} - body = output[:pos] - try: - status = int(output[pos + len(marker):].strip()[-3:]) - except ValueError: - status = 0 - try: - parsed = json.loads(body) if body.strip() else None - except json.JSONDecodeError: - parsed = None - return {"ok": proc.returncode == 0 and 200 <= status < 300, "httpStatus": status, "json": parsed, "body": body, "stderr": text(proc.stderr)} - -def curl_api(method, path, bearer=None, payload=None): - body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8") - script = r''' -set -eu -method="$1" -url="$2" -token="\${3:-}" -tmp="$(mktemp)" -trap 'rm -f "$tmp"' EXIT -cat > "$tmp" -if [ -n "$token" ] && [ "$method" = GET ] && [ ! -s "$tmp" ]; then - curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "Authorization: Bearer $token" "$url" -elif [ -n "$token" ]; then - curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "Authorization: Bearer $token" --data-binary @"$tmp" "$url" -elif [ "$method" = GET ] && [ ! -s "$tmp" ]; then - curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" "$url" -else - curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url" -fi -''' - if RUNTIME_MODE == "host-docker": - proc = run(["sh", "-c", script, "sh", method, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}{path}", bearer or ""], body) - else: - proc = run(["kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or ""], body) - return parse_curl(proc) - -def data_of(response, label): - parsed = response.get("json") - code = parsed.get("code") if isinstance(parsed, dict) else None - if response.get("ok") is not True or (code is not None and code != 0): - message = parsed.get("message") if isinstance(parsed, dict) else text(response.get("body"), 500) - raise RuntimeError(f"{label} failed: http={response.get('httpStatus')} message={message}") - return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed - -def items_of(data): - if isinstance(data, list): - return data - if isinstance(data, dict): - for key in ("items", "groups", "accounts"): - if isinstance(data.get(key), list): - return data[key] - return [] - -def login(): - email = config_value("sub2api-config", "ADMIN_EMAIL", PAYLOAD["adminEmailDefault"]) - password = secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD") - if not password: - raise RuntimeError("ADMIN_PASSWORD missing") - data = data_of(curl_api("POST", "/api/v1/auth/login", payload={"email": email, "password": password}), "admin login") - token = None - if isinstance(data, dict): - token = data.get("access_token") or data.get("token") - if not token: - raise RuntimeError("admin login response has no token") - return token - -def list_groups(token, platform=None): - suffix = "?platform=" + quote(str(platform), safe="") if platform else "" - return items_of(data_of(curl_api("GET", "/api/v1/admin/groups/all" + suffix, bearer=token), "list groups")) - -def list_group_accounts(token, group_id, platform=None): - path = f"/api/v1/admin/accounts?group_id={group_id}&page=1&page_size=500" - if platform: - path += "&platform=" + quote(str(platform), safe="") - return items_of(data_of(curl_api("GET", path, bearer=token), "list group accounts")) - -def account_detail(token, account_id): - return data_of(curl_api("GET", f"/api/v1/admin/accounts/{account_id}", bearer=token), "get account") - -def normalize_policy(credentials): - if not isinstance(credentials, dict): - credentials = {} - enabled = credentials.get("temp_unschedulable_enabled") is True - raw = credentials.get("temp_unschedulable_rules") - if isinstance(raw, str): - try: - raw = json.loads(raw) - except json.JSONDecodeError: - raw = [] - rules = raw if isinstance(raw, list) else [] - codes = sorted(set(item.get("error_code") for item in rules if isinstance(item, dict) and isinstance(item.get("error_code"), int))) - return {"enabled": enabled, "ruleCount": len(rules), "statusCodes": codes, "rules": rules} - -def template_summary(template): - policy = normalize_policy(template.get("credentials")) - return {"id": template.get("id"), "kind": template.get("kind"), "description": template.get("description"), **{key: policy[key] for key in ("enabled", "ruleCount", "statusCodes")}} - -def matching_template(credentials): - current = normalize_policy(credentials) - for template in PAYLOAD["templates"]: - if normalize_policy(template.get("credentials")) == current: - return template.get("id") - return None - -def account_summary(detail, include_rules=False): - credentials = detail.get("credentials") if isinstance(detail.get("credentials"), dict) else {} - policy = normalize_policy(credentials) - name = detail.get("name") - management = "yaml-managed" if name in PAYLOAD["managedAccountNames"] else ("yaml-protected-manual" if name in PAYLOAD["protectedManualAccountNames"] else "runtime-manual") - result = { - "accountName": name, - "accountId": detail.get("id"), - "management": management, - "type": detail.get("type"), - "status": detail.get("status"), - "schedulable": detail.get("schedulable"), - "proxyId": detail.get("proxy_id"), - "concurrencyLimit": detail.get("concurrency"), - "currentConcurrency": detail.get("current_concurrency"), - "loadFactor": detail.get("load_factor"), - "priority": detail.get("priority"), - "tempUnschedulable": {key: policy[key] for key in ("enabled", "ruleCount", "statusCodes")}, - "matchingTemplate": matching_template(credentials), - "poolModeEnabled": credentials.get("pool_mode") is True, - "credentialsPrinted": False, - } - if include_rules: - result["tempUnschedulable"]["rules"] = policy["rules"] - return result - -def list_proxies(token): - return items_of(data_of(curl_api("GET", "/api/v1/admin/proxies?page=1&page_size=500", bearer=token), "list proxies")) - -def proxy_summary(token, proxy_id): - if proxy_id is None: - return {"configured": False, "route": "direct", "credentialsPrinted": False} - matches = [item for item in list_proxies(token) if isinstance(item, dict) and item.get("id") == proxy_id] - if len(matches) != 1: - return {"configured": True, "id": proxy_id, "status": "record-not-found", "credentialsPrinted": False} - item = matches[0] - return { - "configured": True, - "route": "account-proxy", - "id": item.get("id"), - "name": item.get("name"), - "protocol": item.get("protocol"), - "host": item.get("host"), - "port": item.get("port"), - "status": item.get("status"), - "fallbackMode": item.get("fallback_mode"), - "lastCheckAt": item.get("last_check_at"), - "lastError": text(item.get("last_error"), 240) if item.get("last_error") else None, - "credentialsPrinted": False, - } - -def proxy_environment_summary(): - rows = [] - for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): - value = app_text(["sh", "-c", f"printenv {name} 2>/dev/null || true"]) - parsed = urlsplit(value) if value else None - rows.append({ - "name": name, - "present": bool(value), - "scheme": parsed.scheme if parsed else None, - "host": parsed.hostname if parsed else None, - "port": parsed.port if parsed else None, - "credentialsRedacted": bool(parsed and (parsed.username or parsed.password)), - }) - no_proxy = app_text(["sh", "-c", "printenv NO_PROXY 2>/dev/null || printenv no_proxy 2>/dev/null || true"]) - return rows, { - "present": bool(no_proxy), - "entryCount": len([item for item in no_proxy.split(",") if item.strip()]), - "hyueapiPresent": any(item.strip() in ("hyueapi.com", ".hyueapi.com") for item in no_proxy.split(",")), - "valuesPrinted": False, - } - -def app_runtime_summary(): - if RUNTIME_MODE == "host-docker": - proc = docker(["inspect", APP_POD]) - if proc.returncode != 0: - return {"runtimeMode": RUNTIME_MODE, "name": APP_POD, "status": "unavailable", "reason": text(proc.stderr, 240)} - values = json.loads(proc.stdout.decode("utf-8")) - item = values[0] if values else {} - state = item.get("State") or {} - host_config = item.get("HostConfig") or {} - networks = ((item.get("NetworkSettings") or {}).get("Networks") or {}) - return { - "runtimeMode": RUNTIME_MODE, - "name": APP_POD, - "state": state.get("Status"), - "health": ((state.get("Health") or {}).get("Status")), - "restartCount": item.get("RestartCount"), - "oomKilled": state.get("OOMKilled"), - "startedAt": state.get("StartedAt"), - "networkMode": host_config.get("NetworkMode"), - "networks": [ - {"name": name, "ipAddress": value.get("IPAddress"), "gateway": value.get("Gateway")} - for name, value in sorted(networks.items()) if isinstance(value, dict) - ], - } - pod = kube_json(["-n", NAMESPACE, "get", "pod", APP_POD], "sub2api pod") - spec = pod.get("spec") or {} - status = pod.get("status") or {} - restarts = sum(item.get("restartCount") or 0 for item in status.get("containerStatuses") or [] if isinstance(item, dict)) - return { - "runtimeMode": RUNTIME_MODE, - "name": APP_POD, - "state": status.get("phase"), - "restartCount": restarts, - "oomKilled": None, - "startedAt": status.get("startTime"), - "networkMode": "host" if spec.get("hostNetwork") is True else "pod", - "nodeName": spec.get("nodeName"), - "podIp": status.get("podIP"), - "dnsPolicy": spec.get("dnsPolicy"), - } - -def dns_summary(): - content = app_text(["cat", "/etc/resolv.conf"]) - nameservers = [] - search = [] - options = [] - for line in content.splitlines(): - fields = line.strip().split() - if not fields or fields[0].startswith("#"): - continue - if fields[0] == "nameserver": nameservers.extend(fields[1:]) - elif fields[0] == "search": search.extend(fields[1:]) - elif fields[0] == "options": options.extend(fields[1:]) - return {"nameservers": nameservers, "search": search, "options": options} - -def default_route_summary(): - content = app_text(["cat", "/proc/net/route"]) - for line in content.splitlines()[1:]: - fields = line.split() - if len(fields) < 4 or fields[1] != "00000000": - continue - try: - gateway = socket.inet_ntoa(struct.pack("= 5 and peer_fields[0] == "ESTAB": - peers.append(peer_fields[4]) - udp_proc = run(["ss", "-unpH"]) - if udp_proc.returncode == 0: - for udp_line in udp_proc.stdout.decode("utf-8", errors="replace").splitlines(): - if f"pid={pid}," not in udp_line: - continue - udp_fields = udp_line.split() - if len(udp_fields) >= 5 and udp_fields[4] not in ("*:*", "0.0.0.0:*"): - udp_peers.append(udp_fields[4]) - cgroup_proc = run(["cat", f"/proc/{pid}/cgroup"]) - cgroup_text = cgroup_proc.stdout.decode("utf-8", errors="replace") if cgroup_proc.returncode == 0 else "" - unit_match = re.search(r'/([^/]+\\.service)(?:/|$)', cgroup_text) - systemd_unit = unit_match.group(1) if unit_match else None - if systemd_unit: - show_proc = run(["systemctl", "show", systemd_unit, "--property=ActiveState", "--property=SubState", "--property=NRestarts"]) - if show_proc.returncode == 0: - service_values = dict(line.split("=", 1) for line in show_proc.stdout.decode("utf-8", errors="replace").splitlines() if "=" in line) - active_state = "/".join(value for value in (service_values.get("ActiveState"), service_values.get("SubState")) if value) - service_restarts = service_values.get("NRestarts") - rows.append({ - "localEndpoint": local_endpoint, - "processName": process_name, - "pid": pid, - "executable": executable, - "establishedPeers": sorted(set(peers))[:12], - "udpPeers": sorted(set(udp_peers))[:12], - "systemdUnit": systemd_unit, - "activeState": active_state, - "serviceRestarts": service_restarts, - }) - return {"status": "available", "listeners": rows, "credentialsPrinted": False} - -def proxy_service_evidence(owner): - listeners = owner.get("listeners") if isinstance(owner, dict) else [] - units = sorted(set(item.get("systemdUnit") for item in listeners if isinstance(item, dict) and item.get("systemdUnit"))) - if not units: - return {"status": "not-applicable", "buckets": [], "samples": []} - buckets = {} - samples = [] - for unit in units: - proc = run(["journalctl", "--unit", unit, "--since", since_start_time(PAYLOAD.get("since")), "--no-pager", "--output", "json", "--lines", "2000"]) - if proc.returncode != 0: - continue - for line in proc.stdout.decode("utf-8", errors="replace").splitlines(): - try: - item = json.loads(line) - except json.JSONDecodeError: - continue - message = str(item.get("MESSAGE") or "") - lowered = message.lower() - priority = int(item.get("PRIORITY")) if str(item.get("PRIORITY") or "").isdigit() else 7 - cause = None - if "reconnect" in lowered or "retry" in lowered: cause = "reconnect" - elif "disconnect" in lowered or "closed" in lowered: cause = "disconnected" - elif "timeout" in lowered or "deadline" in lowered: cause = "timeout" - elif "error" in lowered or "failed" in lowered or priority <= 3: cause = "service-error" - if cause is None: - continue - buckets[cause] = buckets.get(cause, 0) + 1 - if len(samples) < (30 if PAYLOAD.get("full") is True else 12): - samples.append({"at": item.get("__REALTIME_TIMESTAMP"), "cause": cause, "message": text(message, 400)}) - return { - "status": "available", - "units": units, - "buckets": [{"cause": key, "count": value} for key, value in sorted(buckets.items(), key=lambda pair: (-pair[1], pair[0]))], - "samples": samples, - } - -def classify_network_message(message, status_code): - lowered = message.lower() - checks = ( - ("dns-failure", ("no such host", "name resolution", "temporary failure in name resolution")), - ("proxy-connect-failure", ("proxyconnect", "connect tunnel", "http connect")), - ("connection-reset", ("connection reset", "reset by peer")), - ("unexpected-eof", ("unexpected eof", " eof")), - ("broken-pipe", ("broken pipe",)), - ("connection-refused", ("connection refused",)), - ("network-unreachable", ("network is unreachable", "no route to host")), - ("transport-timeout", ("i/o timeout", "tls handshake timeout", "context deadline exceeded")), - ("context-canceled", ("context canceled", "context cancelled")), - ("stream-incomplete", ("missing terminal event", "stream usage incomplete", "response failed")), - ) - for cause, markers in checks: - if any(marker in lowered for marker in markers): - return cause - return "upstream-503" if status_code == 503 or " 503" in lowered else None - -def recent_network_evidence(account_id): - lines = runtime_log_lines() - buckets = {} - samples = [] - matched = 0 - sample_limit = 30 if PAYLOAD.get("full") is True else 12 - for raw in lines: - candidate = raw[raw.find("{"):] if "{" in raw else raw - try: - item = json.loads(candidate) - except json.JSONDecodeError: - item = {} - extra = item.get("extra") if isinstance(item.get("extra"), dict) else {} - observed_account = item.get("account_id") if item.get("account_id") is not None else extra.get("account_id") - if str(observed_account) != str(account_id): - continue - status_code = item.get("status_code") if isinstance(item.get("status_code"), int) else extra.get("status_code") - if not isinstance(status_code, int): - upstream_status = extra.get("upstream_status") - status_code = upstream_status if isinstance(upstream_status, int) else None - message_parts = [item.get("message"), item.get("error"), extra.get("error"), extra.get("message")] - message = " | ".join(str(value) for value in message_parts if value not in (None, "")) - cause = classify_network_message(message, status_code) - if cause is None: - continue - matched += 1 - buckets[cause] = buckets.get(cause, 0) + 1 - if len(samples) < sample_limit: - samples.append({ - "at": item.get("time") or item.get("created_at") or extra.get("_at"), - "requestId": item.get("request_id") or extra.get("request_id"), - "cause": cause, - "statusCode": status_code, - "message": text(message, 360), - }) - return { - "source": "sub2api-target-runtime-logs", - "window": PAYLOAD.get("since"), - "scannedLines": len(lines), - "matchedEvents": matched, - "buckets": [{"cause": key, "count": value} for key, value in sorted(buckets.items(), key=lambda pair: (-pair[1], pair[0]))], - "samples": samples, - } - -def infrastructure_snapshot(token, detail): - proxy = proxy_summary(token, detail.get("proxy_id")) - proxy_env, no_proxy = proxy_environment_summary() - owner = proxy_endpoint_owner(proxy) - return { - "source": "sub2api-admin-api-and-target-runtime", - "accountProxy": proxy, - "appRuntime": app_runtime_summary(), - "proxyEnvironment": proxy_env, - "noProxy": no_proxy, - "dns": dns_summary(), - "defaultRoute": default_route_summary(), - "interfaces": interface_summaries(), - "proxySockets": proxy_socket_summary(proxy), - "proxyEndpointOwner": owner, - "proxyServiceEvidence": proxy_service_evidence(owner), - "recentNetworkEvidence": recent_network_evidence(detail.get("id")), - "attribution": "The account proxy record controls this account's transport. Container proxy environment is reported separately. Interface counters and sockets are query-time snapshots; no upstream account test or application request is sent.", - "valuesPrinted": False, - } - -def policy_summary(policy, include_rules=False): - keys = ("enabled", "ruleCount", "statusCodes", "rules") if include_rules else ("enabled", "ruleCount", "statusCodes") - return {key: policy[key] for key in keys} - -def find_account(token, accounts, selector): - exact = [item for item in accounts if str(item.get("id")) == selector or item.get("name") == selector] - if len(exact) != 1: - raise RuntimeError("account-not-found-or-ambiguous: " + selector) - return account_detail(token, exact[0]["id"]) - -def runtime_log_lines(): - since = PAYLOAD.get("since", "24h") - tail = str(PAYLOAD.get("tail", 50000)) - if RUNTIME_MODE == "host-docker": - proc = docker(["logs", "--since", since, "--tail", tail, APP_POD]) - else: - proc = kubectl(["-n", NAMESPACE, "logs", APP_POD, "--since=" + since, "--tail=" + tail]) - if proc.returncode != 0: - raise RuntimeError("read Sub2API runtime logs failed: " + text(proc.stderr)) - output = proc.stdout + b"\\n" + proc.stderr - return output.decode("utf-8", errors="replace").splitlines() - -def app_exec(args): - if RUNTIME_MODE == "host-docker": - return docker(["exec", APP_POD, *args]) - return kubectl(["-n", NAMESPACE, "exec", APP_POD, "--", *args]) - -def app_text(args): - proc = app_exec(args) - return proc.stdout.decode("utf-8", errors="replace").strip() if proc.returncode == 0 else "" - -def config_scalar(path, key): - script = r'''awk -v wanted="$2" ' -BEGIN { in_gateway=0 } -/^gateway:[[:space:]]*($|#)/ { in_gateway=1; next } -in_gateway && /^[^[:space:]#]/ { exit } -in_gateway && $0 ~ "^[[:space:]]+" wanted ":[[:space:]]*" { - value=$0 - sub("^[[:space:]]+" wanted ":[[:space:]]*", "", value) - sub("[[:space:]]*#.*$", "", value) - gsub("^[[:space:]\\\"'\''']+|[[:space:]\\\"'\''']+$", "", value) - print value - exit -}' "$1"''' - return app_text(["sh", "-c", script, "sh", path, key]) - -def runtime_response_header_timeout(): - env_openai = app_text(["sh", "-c", "printenv GATEWAY_OPENAI_RESPONSE_HEADER_TIMEOUT 2>/dev/null || true"]) - env_generic = app_text(["sh", "-c", "printenv GATEWAY_RESPONSE_HEADER_TIMEOUT 2>/dev/null || true"]) - config_file = app_text(["sh", "-c", "printenv CONFIG_FILE 2>/dev/null || true"]) - paths = [value for value in (config_file, "/app/data/config.yaml", "/etc/sub2api/config.yaml", "/app/config.yaml") if value] - config_openai = "" - config_generic = "" - config_source = None - for path in dict.fromkeys(paths): - openai_value = config_scalar(path, "openai_response_header_timeout") - generic_value = config_scalar(path, "response_header_timeout") - if openai_value or generic_value: - config_openai = openai_value - config_generic = generic_value - config_source = path - break - def non_negative_int(value): - try: - parsed = int(str(value).strip()) - return parsed if parsed >= 0 else None - except (TypeError, ValueError): - return None - openai_configured = non_negative_int(config_openai) - openai_env = non_negative_int(env_openai) - generic_configured = non_negative_int(config_generic) - generic_env = non_negative_int(env_generic) - effective = openai_configured if openai_configured is not None else openai_env if openai_env is not None else 0 - source = "config-file:" + config_source if openai_configured is not None else "process-env" if openai_env is not None else "sub2api-default" - generic = generic_configured if generic_configured is not None else generic_env if generic_env is not None else 600 - return { - "effectiveSeconds": effective, - "disabled": effective == 0, - "source": source, - "genericResponseHeaderTimeoutSeconds": generic, - "genericAppliesToOpenAI": False, - "configFileChecked": config_source, - "valuesPrinted": False, - } - -def compact_error(value): - if not isinstance(value, str): - return "" - compact = " ".join(value.replace("\x00", "").split()) - compact = re.sub(r"(?i)\\bBearer\\s+[A-Za-z0-9._~+/=-]+", "Bearer [REDACTED]", compact) - compact = re.sub(r"\\bsk-[A-Za-z0-9_-]{8,}\\b", "sk-[REDACTED]", compact) - compact = re.sub(r"(?i)\\b(api[_ -]?key|token)[=:]\\s*[^\\s,;]+", r"\\1=[REDACTED]", compact) - return compact[:240] - -def since_start_time(value): - match = re.fullmatch(r"(\\d+)([mhd])", str(value or "")) - if not match: - raise RuntimeError("unsupported runtime error window: " + str(value)) - amount = int(match.group(1)) - unit = match.group(2) - duration = timedelta(minutes=amount) if unit == "m" else timedelta(hours=amount) if unit == "h" else timedelta(days=amount) - return (datetime.now(timezone.utc) - duration).isoformat().replace("+00:00", "Z") - -def ops_error_cause(value): - lower = value.lower() - markers = { - "concurrency": ("concurrency limit", "concurrent request", "too many concurrent", "maximum concurrency", "max concurrency"), - "overload": ("overloaded", "server busy", "temporarily unavailable", "capacity exhausted"), - "model-route-unavailable": ("no available channel for model", "unsupported model", "model not found", "model_not_found"), - "context-window": ("context window", "context length", "maximum context"), - "continuation-payload": ("encrypted_content", "missing_required_parameter", "missing required parameter"), - "invalid-request": ("invalid_request", "invalid request"), - "rate-limit": ("rate limit", "too many requests"), - "access-denied": ("forbidden", "access denied", "permission denied"), - "not-found": ("not found", "does not exist"), - "incomplete-stream": ("ended before a terminal event", "missing terminal event", "before response.completed"), - "transport": ("do request failed", "unexpected eof", "connection reset", "stream read error", "stream closed", "internal_error; received from peer"), - "upstream-api-key-validation": ("failed to validate api key", "api key validation failed"), - "authentication": ("unauthorized", "invalid token", "token revoked", "invalid_grant"), - "generic-upstream-failure": ("upstream request failed", "upstream gateway error"), - } - for cause, values in markers.items(): - if any(marker in lower for marker in values): - return cause - if re.match(r"recovered upstream error \\d+\\b", lower): - return "status-only-upstream-failure" - return "unknown" - -def ops_error_analysis(token, account, log_statuses): - account_id = account.get("id") - start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") - path = f"/api/v1/admin/ops/upstream-errors?account_id={account_id}&view=all&page=1&page_size=500&start_time={start_time}" - response = curl_api("GET", path, bearer=token) - try: - data = data_of(response, "list account ops errors") - except Exception as exc: - return { - "source": "ops-upstream-errors", - "status": "unavailable", - "logObservedStatusCodes": log_statuses, - "reason": compact_error(str(exc)), - "assessment": "insufficient-evidence", - } - rows = items_of(data) - total = data.get("total", len(rows)) if isinstance(data, dict) else len(rows) - detail_limit = 24 if PAYLOAD.get("full") is True else 8 - analyzed = [] - for row in rows: - message = compact_error(row.get("message") if isinstance(row, dict) else "") - analyzed.append({"row": row, "evidence": message, "cause": ops_error_cause(message)}) - for entry in analyzed[:detail_limit]: - row = entry["row"] - error_id = row.get("id") if isinstance(row, dict) else None - if not isinstance(error_id, int): - continue - detail_response = curl_api("GET", f"/api/v1/admin/ops/upstream-errors/{error_id}", bearer=token) - try: - detail = data_of(detail_response, "get account 500 ops error") - except Exception: - continue - fields = [] - for key in ("message", "upstream_error_message", "upstream_error_detail", "error_body", "upstream_errors"): - value = detail.get(key) if isinstance(detail, dict) else None - if isinstance(value, str) and value.strip(): - fields.append(value) - evidence = compact_error(" ".join(fields)) - entry["evidence"] = evidence - entry["cause"] = ops_error_cause(evidence) - - causes = {} - statuses = {} - messages = {} - models = {} - hours = {} - stream_modes = {"stream": 0, "synchronous": 0, "unknown": 0} - concurrency_count = 0 - for entry in analyzed: - row = entry["row"] - error_id = row.get("id") if isinstance(row, dict) else None - evidence = entry["evidence"] - cause = entry["cause"] - causes[cause] = causes.get(cause, 0) + 1 - list_message = compact_error(row.get("message") if isinstance(row, dict) else "") - if list_message: - messages[list_message] = messages.get(list_message, 0) + 1 - status_code = row.get("status_code") if isinstance(row, dict) else None - if isinstance(status_code, int): - statuses[str(status_code)] = statuses.get(str(status_code), 0) + 1 - if cause == "concurrency": - concurrency_count += 1 - model = str(row.get("model") or "unknown") if isinstance(row, dict) else "unknown" - models[model] = models.get(model, 0) + 1 - created_at = str(row.get("created_at") or "") if isinstance(row, dict) else "" - hour = created_at[:13] + ":00" if len(created_at) >= 13 else "unknown" - hours[hour] = hours.get(hour, 0) + 1 - stream = row.get("stream") if isinstance(row, dict) else None - stream_key = "stream" if stream is True else "synchronous" if stream is False else "unknown" - stream_modes[stream_key] += 1 - sample = { - "errorId": error_id, - "createdAt": row.get("created_at") if isinstance(row, dict) else None, - "requestId": row.get("request_id") if isinstance(row, dict) else None, - "model": row.get("model") if isinstance(row, dict) else None, - "stream": row.get("stream") if isinstance(row, dict) else None, - "cause": cause, - "evidence": evidence, - } - entry["sample"] = sample - analyzed_count = len(analyzed) - if analyzed_count == 0: - assessment = "insufficient-evidence" - elif concurrency_count == analyzed_count: - assessment = "concurrency-related" - elif concurrency_count > 0: - assessment = "mixed" - else: - assessment = "no-direct-concurrency-evidence" - dominant_cause = None - dominant_count = 0 - if causes: - dominant_cause, dominant_count = max(causes.items(), key=lambda pair: pair[1]) - dominant_share = round(dominant_count / analyzed_count, 4) if analyzed_count > 0 else None - if dominant_cause == "upstream-api-key-validation": - finding = "The upstream gateway returned HTTP 500 while validating its inbound API key; investigate its key lookup and storage dependencies first." - elif dominant_cause == "concurrency": - finding = "Direct concurrency-limit messages dominate the observed Ops records." - elif dominant_cause is not None: - finding = "The dominant observed Ops cause is " + dominant_cause + "." - else: - finding = "No structured HTTP 500 Ops records were available for cause analysis." - if concurrency_count > 0: - concurrency_action = "consider-lower-limit-after-controlled-comparison" - else: - concurrency_action = "do-not-lower-limit-as-primary-fix" - ranked_messages = sorted(messages.items(), key=lambda pair: (-pair[1], pair[0])) - message_limit = 20 if PAYLOAD.get("full") is True else 5 - return { - "source": "ops-upstream-errors", - "status": "available", - "logObservedStatusCodes": log_statuses, - "opsRecordCount": total, - "analyzedRecordCount": analyzed_count, - "detailRecordCount": min(detail_limit, analyzed_count), - "recordsTruncated": total > analyzed_count, - "statusBuckets": [{"statusCode": int(code), "count": count} for code, count in sorted(statuses.items(), key=lambda pair: int(pair[0]))], - "causeBuckets": [{"cause": cause, "count": count} for cause, count in sorted(causes.items())], - "messageBuckets": [{"message": message, "count": count} for message, count in ranked_messages[:message_limit]], - "concurrencyRelatedRecordCount": concurrency_count, - "assessment": assessment, - "rootCause": { - "dominantCause": dominant_cause, - "dominantCount": dominant_count, - "dominantShare": dominant_share, - "finding": finding, - }, - "errorRateEvidence": { - "errorRecordCount": total, - "requestAttemptDenominator": None, - "errorRate": None, - "reason": "The available Ops endpoint lists errors but does not provide all routing attempts for this account.", - }, - "concurrencyRecommendation": { - "action": concurrency_action, - "reason": "No error-time concurrency history is available; only explicit concurrency error text is treated as direct evidence.", - }, - "runtimeConcurrency": { - "limit": account.get("concurrency"), - "current": account.get("current_concurrency"), - "loadFactor": account.get("load_factor"), - "historicalAtErrorAvailable": False, - }, - **({ - "modelBuckets": [{"model": model, "count": count} for model, count in sorted(models.items(), key=lambda pair: (-pair[1], pair[0]))], - "hourBuckets": [{"hour": hour, "count": count} for hour, count in sorted(hours.items())], - "streamBuckets": [{"mode": mode, "count": count} for mode, count in stream_modes.items() if count > 0], - "samples": [entry["sample"] for entry in analyzed[:detail_limit]], - } if PAYLOAD.get("full") is True else { - "samples": [entry["sample"] for entry in analyzed[:1]], - }), - "disclosure": "Ops records are independently sampled evidence and are not positionally joined to account_upstream_error log lines.", - } - -def gateway_request_path(value): - path = str(value or "").split("?", 1)[0].rstrip("/") - return path in ( - "/responses", "/v1/responses", "/responses/compact", "/v1/responses/compact", - "/messages", "/v1/messages", "/chat/completions", "/v1/chat/completions", - ) - -def native_ops_snapshot(token, group_id, platform=None): - start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") - platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" - base_query = f"group_id={group_id}&start_time={start_time}{platform_query}" - - def optional(path, label): - response = curl_api("GET", path, bearer=token) - try: - return {"status": "available", "data": data_of(response, label)} - except Exception as exc: - return {"status": "unavailable", "reason": compact_error(str(exc))} - - overview_result = optional(f"/api/v1/admin/ops/dashboard/overview?{base_query}&query_mode=raw", "get ops dashboard overview") - availability_result = optional(f"/api/v1/admin/ops/account-availability?group_id={group_id}{platform_query}", "get ops account availability") - concurrency_result = optional(f"/api/v1/admin/ops/concurrency?group_id={group_id}{platform_query}", "get ops concurrency") - sink_result = optional("/api/v1/admin/ops/system-logs/health", "get ops system log health") - - overview = overview_result.get("data") if overview_result.get("status") == "available" else None - overview_summary = None - if isinstance(overview, dict): - overview_summary = { - key: overview.get(key) - for key in ( - "start_time", "end_time", "health_score", "success_count", "error_count_total", - "business_limited_count", "error_count_sla", "request_count_total", "request_count_sla", - "error_rate", "upstream_error_rate", "upstream_error_count_excl_429_529", - "upstream_429_count", "upstream_529_count", - "duration", "ttft", - ) - } - - availability = availability_result.get("data") if availability_result.get("status") == "available" else None - availability_summary = None - if isinstance(availability, dict): - group_values = availability.get("group") if isinstance(availability.get("group"), dict) else {} - account_values = availability.get("account") if isinstance(availability.get("account"), dict) else {} - selected_group = group_values.get(str(group_id)) or group_values.get(group_id) - unavailable = [] - for value in account_values.values(): - if not isinstance(value, dict) or value.get("group_id") != group_id or value.get("is_available") is True: - continue - unavailable.append({ - "accountId": value.get("account_id"), - "accountName": value.get("account_name"), - "status": value.get("status"), - "rateLimited": value.get("is_rate_limited"), - "rateLimitRemainingSeconds": value.get("rate_limit_remaining_sec"), - "overloaded": value.get("is_overloaded"), - "overloadRemainingSeconds": value.get("overload_remaining_sec"), - "hasError": value.get("has_error"), - "error": compact_error(value.get("error_message")), - }) - availability_summary = { - "enabled": availability.get("enabled"), - "timestamp": availability.get("timestamp"), - "group": selected_group, - "unavailableAccounts": unavailable[:20 if PAYLOAD.get("full") is True else 8], - "unavailableAccountCount": max( - len(unavailable), - max(0, selected_group.get("total_accounts") - selected_group.get("available_count")) - if isinstance(selected_group, dict) - and isinstance(selected_group.get("total_accounts"), int) - and isinstance(selected_group.get("available_count"), int) - else 0, - ), - "unavailableAccountDetailsComplete": ( - not isinstance(selected_group, dict) - or not isinstance(selected_group.get("total_accounts"), int) - or not isinstance(selected_group.get("available_count"), int) - or len(unavailable) >= max(0, selected_group.get("total_accounts") - selected_group.get("available_count")) - ), - } - - concurrency = concurrency_result.get("data") if concurrency_result.get("status") == "available" else None - concurrency_summary = None - if isinstance(concurrency, dict): - group_values = concurrency.get("group") if isinstance(concurrency.get("group"), dict) else {} - account_values = concurrency.get("account") if isinstance(concurrency.get("account"), dict) else {} - selected_group = group_values.get(str(group_id)) or group_values.get(group_id) - loaded = [] - for value in account_values.values(): - if not isinstance(value, dict) or value.get("group_id") != group_id: - continue - if (value.get("current_in_use") or 0) <= 0 and (value.get("waiting_in_queue") or 0) <= 0: - continue - loaded.append({ - "accountId": value.get("account_id"), - "accountName": value.get("account_name"), - "currentInUse": value.get("current_in_use"), - "maxCapacity": value.get("max_capacity"), - "loadPercentage": value.get("load_percentage"), - "waitingInQueue": value.get("waiting_in_queue"), - }) - concurrency_summary = { - "enabled": concurrency.get("enabled"), - "timestamp": concurrency.get("timestamp"), - "group": selected_group, - "activeAccounts": sorted(loaded, key=lambda item: (-(item.get("loadPercentage") or 0), str(item.get("accountName") or "")))[:20 if PAYLOAD.get("full") is True else 8], - } - - sink = sink_result.get("data") if sink_result.get("status") == "available" else None - return { - "source": "sub2api-native-admin-ops", - "overview": {"status": overview_result.get("status"), "summary": overview_summary, "reason": overview_result.get("reason")}, - "accountAvailability": {"status": availability_result.get("status"), "summary": availability_summary, "reason": availability_result.get("reason")}, - "concurrency": {"status": concurrency_result.get("status"), "summary": concurrency_summary, "reason": concurrency_result.get("reason")}, - "systemLogSink": {"status": sink_result.get("status"), "health": sink if isinstance(sink, dict) else None, "reason": sink_result.get("reason")}, - } - -def native_request_timing(token, group_id, platform, overview): - start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") - platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" - base = f"group_id={group_id}&start_time={start_time}{platform_query}" - all_counts = {} - error_counts = {} - status = "available" - reason = None - duration = overview.get("duration") if isinstance(overview.get("duration"), dict) else {} - ttft = overview.get("ttft") if isinstance(overview.get("ttft"), dict) else {} - ttft_p99 = ttft.get("p99_ms") - observed_floor = max(60, ((ttft_p99 + 29999) // 15000) * 15) if isinstance(ttft_p99, int) else None - candidate_seconds = sorted(set([30, 45, 60] + ([observed_floor] if isinstance(observed_floor, int) else []))) - error_duration_response = curl_api("GET", f"/api/v1/admin/ops/requests?{base}&page=1&page_size=1&kind=error&min_duration_ms=0", bearer=token) - try: - error_duration_data = data_of(error_duration_response, "get error duration coverage") - error_duration_record_count = error_duration_data.get("total") if isinstance(error_duration_data, dict) else None - except Exception as exc: - error_duration_record_count = None - status = "unavailable" - reason = compact_error(str(exc)) - for seconds in candidate_seconds: - threshold = seconds * 1000 - for kind, target in ((None, all_counts), ("error", error_counts)): - kind_query = "&kind=error" if kind else "" - response = curl_api("GET", f"/api/v1/admin/ops/requests?{base}&page=1&page_size=1&min_duration_ms={threshold}{kind_query}", bearer=token) - try: - data = data_of(response, "get long request count") - value = data.get("total") if isinstance(data, dict) else None - target[threshold] = None if kind == "error" and error_duration_record_count == 0 else value - except Exception as exc: - status = "unavailable" - reason = compact_error(str(exc)) - target[threshold] = None - candidates = [] - for seconds in candidate_seconds: - headroom = seconds * 1000 - ttft_p99 if isinstance(ttft_p99, int) else None - candidates.append({ - "seconds": seconds, - "observedFloor": seconds == observed_floor, - "ttftP99HeadroomMs": headroom, - "ttftP99Below": headroom >= 0 if isinstance(headroom, int) else None, - "totalDurationAtLeastCount": all_counts.get(seconds * 1000), - "errorDurationAtLeastCount": error_counts.get(seconds * 1000), - "assessment": "observed-ttft-p99-plus-15s-rounded" if seconds == observed_floor else "ttft-p99-within-candidate" if isinstance(headroom, int) and headroom >= 0 else "ttft-p99-exceeds-candidate" if isinstance(headroom, int) else "insufficient-ttft-evidence", - }) - since = str(PAYLOAD.get("since") or "") - token_range = {"30m": "30m", "60m": "1h", "1h": "1h", "24h": "1d", "1d": "1d"}.get(since) - model_timing = [] - model_timing_status = "unsupported-window" - if token_range: - response = curl_api("GET", f"/api/v1/admin/ops/dashboard/openai-token-stats?group_id={group_id}&time_range={token_range}&top_n=100{platform_query}", bearer=token) - try: - data = data_of(response, "get OpenAI model token stats") - rows = items_of(data) - for row in rows: - if not isinstance(row, dict): - continue - request_count = row.get("request_count") - first_count = row.get("requests_with_first_token") - model_timing.append({ - "model": row.get("model"), "requestCount": request_count, - "requestsWithFirstToken": first_count, - "firstTokenCoverage": round(first_count / request_count, 6) if isinstance(first_count, int) and isinstance(request_count, int) and request_count > 0 else None, - "avgFirstTokenMs": row.get("avg_first_token_ms"), "avgDurationMs": row.get("avg_duration_ms"), - "avgTokensPerSec": row.get("avg_tokens_per_sec"), - }) - model_timing.sort(key=lambda item: (-(item.get("avgFirstTokenMs") or -1), str(item.get("model") or ""))) - model_timing_status = "available" - except Exception as exc: - model_timing_status = "unavailable" - reason = compact_error(str(exc)) - return { - "status": status, - "reason": reason, - "source": "sub2api-native-admin-ops-dashboard-and-requests", - "ttft": ttft, - "duration": duration, - "observedFloorSeconds": observed_floor, - "errorDurationRecordCount": error_duration_record_count, - "modelTimingStatus": model_timing_status, - "modelTimingWindow": token_range, - "modelTiming": model_timing[:20 if PAYLOAD.get("full") is True else 8], - "candidates": candidates, - "attribution": "TTFT covers successful streaming requests with first_token_ms. Total and error duration are end-to-end request durations, not response-header wait or guaranteed failover lead time. The observed floor is TTFT P99 plus 15 seconds rounded up to 15 seconds; it is not a deployment recommendation. A dash in ERROR_DUR>= means error duration coverage is unavailable.", - } - -def native_customer_error_profile(token, group_id, platform): - start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") - platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" - page = 1 - page_size = 500 - rows = [] - total = None - try: - while True: - response = curl_api("GET", f"/api/v1/admin/ops/request-errors?group_id={group_id}&start_time={start_time}&view=all&page={page}&page_size={page_size}{platform_query}", bearer=token) - data = data_of(response, "get customer-visible request errors") - page_rows = items_of(data) - rows.extend(page_rows) - if total is None and isinstance(data, dict) and isinstance(data.get("total"), int): - total = data.get("total") - if not page_rows or len(page_rows) < page_size or (isinstance(total, int) and len(rows) >= total): - break - page += 1 - except Exception as exc: - return {"status": "unavailable", "source": "sub2api-native-admin-ops-request-errors", "reason": compact_error(str(exc))} - if total is None: - total = len(rows) - def buckets(key_fn): - counts = {} - for row in rows: - value = key_fn(row) if isinstance(row, dict) else None - label = str(value) if value not in (None, "") else "unknown" - counts[label] = counts.get(label, 0) + 1 - return [{"value": key, "count": count} for key, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:20 if PAYLOAD.get("full") is True else 8]] - samples = [] - account_metrics = {} - seen = set() - for row in rows: - if not isinstance(row, dict): - continue - account_id = row.get("account_id") - account_key = str(account_id) if account_id is not None else "unknown" - if account_key not in account_metrics: - account_metrics[account_key] = { - "accountId": account_id, "customerErrorEvents": 0, "customerErrorRequestIds": set(), "statusCodes": {}, - } - metric = account_metrics[account_key] - metric["customerErrorEvents"] += 1 - request_id = row.get("request_id") - if isinstance(request_id, str) and request_id: - metric["customerErrorRequestIds"].add(request_id) - status_code = row.get("status_code") - if isinstance(status_code, int): - status_key = str(status_code) - metric["statusCodes"][status_key] = metric["statusCodes"].get(status_key, 0) + 1 - if not request_id: - continue - key = (row.get("requested_model") or row.get("model"), row.get("status_code"), row.get("account_id"), row.get("stream"), row.get("request_path") or row.get("inbound_endpoint")) - if key in seen: - continue - seen.add(key) - samples.append({ - "requestId": row.get("request_id"), "model": key[0], "statusCode": key[1], "accountId": key[2], - "mode": "stream" if key[3] is True else "sync" if key[3] is False else "unknown", "path": key[4], - "message": compact_error(row.get("message") or row.get("error_message")), - }) - if len(samples) >= (30 if PAYLOAD.get("full") is True else 15): - break - return { - "status": "available", "source": "sub2api-native-admin-ops-request-errors", "total": total, - "returned": len(rows), "pagesRead": page, "recordsTruncated": False, - "modelBuckets": buckets(lambda row: row.get("requested_model") or row.get("model")), - "accountBuckets": buckets(lambda row: row.get("account_id")), - "statusBuckets": buckets(lambda row: row.get("status_code")), - "streamBuckets": buckets(lambda row: "stream" if row.get("stream") is True else "sync" if row.get("stream") is False else "unknown"), - "messageBuckets": buckets(lambda row: compact_error(row.get("message") or row.get("error_message"))), - "phaseBuckets": buckets(lambda row: row.get("phase")), - "endpointBuckets": buckets(lambda row: row.get("inbound_endpoint") or row.get("request_path")), - "pathBuckets": buckets(lambda row: row.get("request_path") or row.get("inbound_endpoint")), - "accountMetrics": [ - { - "accountId": item["accountId"], "customerErrorEvents": item["customerErrorEvents"], - "customerErrorRequests": len(item["customerErrorRequestIds"]), - "_requestIds": sorted(item["customerErrorRequestIds"]), - "statusBuckets": [ - {"statusCode": int(code), "count": count} - for code, count in sorted(item["statusCodes"].items(), key=lambda pair: int(pair[0])) - ], - } - for item in sorted(account_metrics.values(), key=lambda item: int(item.get("accountId") or 0)) - ], - "samples": samples, - } - -def parse_native_time(value): - if not isinstance(value, str) or not value: - return None - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc) - except ValueError: - return None - -def metric_percentile(values, fraction): - ordered = sorted(value for value in values if isinstance(value, (int, float)) and value >= 0) - if not ordered: - return None - position = (len(ordered) - 1) * fraction - lower = int(position) - upper = min(lower + 1, len(ordered) - 1) - return round(ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)) - -def account_usage_metrics(token, accounts, group_id): - start_at = parse_native_time(since_start_time(PAYLOAD.get("since"))) - end_at = datetime.now(timezone.utc) - result = {} - for account in accounts: - account_id = account.get("id") - key = str(account_id) - rows = [] - page = 1 - status = "available" - reason = None - try: - while True: - path = ( - f"/api/v1/admin/usage?page={page}&page_size=100&group_id={group_id}&account_id={account_id}" - f"&start_date={start_at.date().isoformat()}&end_date={end_at.date().isoformat()}&timezone=UTC" - "&sort_by=created_at&sort_order=desc" - ) - data = data_of(curl_api("GET", path, bearer=token), "list account usage") - page_rows = items_of(data) - reached_start = False - for row in page_rows: - created_at = parse_native_time(row.get("created_at") if isinstance(row, dict) else None) - if created_at is None or created_at < start_at or created_at > end_at: - if created_at is not None and created_at < start_at: - reached_start = True - continue - rows.append(row) - if not page_rows or len(page_rows) < 100 or reached_start: - break - page += 1 - except Exception as exc: - status = "unavailable" - reason = compact_error(str(exc)) - stream_rows = [row for row in rows if isinstance(row, dict) and row.get("stream") is True] - ttft_values = [row.get("first_token_ms") for row in stream_rows if isinstance(row.get("first_token_ms"), int)] - duration_values = [row.get("duration_ms") for row in rows if isinstance(row.get("duration_ms"), int)] - models = {} - for row in rows: - model = str(row.get("model") or "unknown") - models[model] = models.get(model, 0) + 1 - result[key] = { - "status": status, "reason": reason, "source": "sub2api-native-admin-usage", - "pagesRead": page, "successRequests": len(rows), "streamSuccessRequests": len(stream_rows), - "firstTokenSamples": len(ttft_values), - "firstTokenCoverage": round(len(ttft_values) / len(stream_rows), 6) if stream_rows else None, - "ttftP50Ms": metric_percentile(ttft_values, 0.50), "ttftP95Ms": metric_percentile(ttft_values, 0.95), - "ttftP99Ms": metric_percentile(ttft_values, 0.99), "ttftMaxMs": max(ttft_values) if ttft_values else None, - "durationP95Ms": metric_percentile(duration_values, 0.95), - "modelBuckets": [{"model": model, "count": count} for model, count in sorted(models.items(), key=lambda item: (-item[1], item[0]))[:8]], - } - return result - -def reliability_score_points(failure_rate): - if not isinstance(failure_rate, (int, float)): - return None - return round(60 * (1 - min(max(failure_rate, 0), 0.20) / 0.20), 2) - -def latency_score_points(ttft_p95_ms): - if not isinstance(ttft_p95_ms, (int, float)): - return None - return round(25 * (1 - min(max(ttft_p95_ms - 10000, 0), 170000) / 170000), 2) - -def account_quality_scores(token, accounts, group_id, native_ops, effectiveness): - usage_by_account = account_usage_metrics(token, accounts, group_id) - customer_profile = native_ops.get("customerErrors") if isinstance(native_ops.get("customerErrors"), dict) else {} - customer_by_account = {str(item.get("accountId")): item for item in customer_profile.get("accountMetrics") or [] if isinstance(item, dict)} - policy_by_account = {str(item.get("accountId")): item for item in effectiveness.get("accountPolicyMetrics") or [] if isinstance(item, dict)} - availability = ((native_ops.get("accountAvailability") or {}).get("summary") or {}) - unavailable_ids = set(str(item.get("accountId")) for item in availability.get("unavailableAccounts") or [] if isinstance(item, dict)) - rows = [] - for account in accounts: - account_id = account.get("id") - key = str(account_id) - usage = usage_by_account.get(key) or {} - customer = customer_by_account.get(key) or {} - policy = policy_by_account.get(key) or {} - policy_failure_ids = set(policy.pop("_failureRequestIds", []) or []) - customer_failure_ids = set(customer.pop("_requestIds", []) or []) - failure_requests = len(policy_failure_ids | customer_failure_ids) - success_requests = usage.get("successRequests") if isinstance(usage.get("successRequests"), int) else 0 - observed_attempts = success_requests + failure_requests - failure_rate = round(failure_requests / observed_attempts, 6) if observed_attempts > 0 else None - reliability_points = reliability_score_points(failure_rate) - ttft_samples = usage.get("firstTokenSamples") if isinstance(usage.get("firstTokenSamples"), int) else 0 - latency_points = latency_score_points(usage.get("ttftP95Ms")) if ttft_samples >= 5 else None - current_available = key not in unavailable_ids and account.get("status") == "active" and account.get("schedulable") is not False - availability_points = 15 if current_available else 8 if account.get("status") == "active" else 0 - earned = (reliability_points or 0) + (latency_points or 0) + availability_points - available_weight = (60 if reliability_points is not None else 0) + (25 if latency_points is not None else 0) + 15 - score = round(earned / available_weight * 100, 1) if observed_attempts > 0 and available_weight > 0 else None - score_comparable = observed_attempts >= 10 and ttft_samples >= 5 - grade = "insufficient" - if isinstance(score, (int, float)) and (score_comparable or (score < 60 and observed_attempts >= 10)): - grade = "A" if score >= 90 else "B" if score >= 80 else "C" if score >= 70 else "D" if score >= 60 else "E" - assessment = "preferred" if grade == "A" else "healthy" if grade == "B" else "watch" if grade == "C" else "degraded" if grade == "D" else "poor" if grade == "E" else "insufficient-evidence" - confidence = "high" if observed_attempts >= 50 and ttft_samples >= 20 else "medium" if observed_attempts >= 10 and ttft_samples >= 5 else "low" - reasons = [] - if isinstance(failure_rate, (int, float)) and failure_rate >= 0.10: - reasons.append("failure-rate>=10%") - elif isinstance(failure_rate, (int, float)) and failure_rate >= 0.03: - reasons.append("failure-rate>=3%") - if isinstance(usage.get("ttftP95Ms"), int) and usage.get("ttftP95Ms") >= 120000: - reasons.append("ttft-p95>=120s") - if policy.get("failoverRequests", 0) > 0: - reasons.append("upstream-failover-triggered") - if policy.get("sameAccountRetryEvents", 0) > 0: - reasons.append("same-account-retry-observed") - if not current_available: - reasons.append("currently-unavailable") - if ttft_samples < 5: - reasons.append("ttft-evidence-insufficient") - if observed_attempts < 10: - reasons.append("request-evidence-insufficient") - rows.append({ - "accountId": account_id, "accountName": account.get("name"), "status": account.get("status"), - "schedulable": account.get("schedulable"), "currentlyAvailable": current_available, - "score": score, "grade": grade, "assessment": assessment, "confidence": confidence, - "scoreComparable": score_comparable, - "observedAttempts": observed_attempts, "successRequests": success_requests, - "failureRequests": failure_requests, "failureRate": failure_rate, - "streamSuccessRequests": usage.get("streamSuccessRequests"), "firstTokenSamples": ttft_samples, - "firstTokenCoverage": usage.get("firstTokenCoverage"), "ttftP50Ms": usage.get("ttftP50Ms"), - "ttftP95Ms": usage.get("ttftP95Ms"), "ttftP99Ms": usage.get("ttftP99Ms"), "ttftMaxMs": usage.get("ttftMaxMs"), - "durationP95Ms": usage.get("durationP95Ms"), "modelBuckets": usage.get("modelBuckets"), - "customerErrorRequests": customer.get("customerErrorRequests", 0), - "failoverRequests": policy.get("failoverRequests", 0), "failoverRecovered": policy.get("failoverRecovered", 0), - "failoverFailed": policy.get("failoverFailed", 0), "failoverOutcomeMissing": policy.get("failoverOutcomeMissing", 0), - "sameAccountRetryEvents": policy.get("sameAccountRetryEvents", 0), - "tempUnschedulableEvents": policy.get("tempUnschedulableEvents", 0), - "forwardFailedRequests": policy.get("forwardFailedRequests", 0), - "upstreamStatusBuckets": policy.get("upstreamStatusBuckets", []), - "scoreComponents": {"reliability": reliability_points, "latency": latency_points, "availability": availability_points, "availableWeight": available_weight}, - "reasons": reasons, "usageStatus": usage.get("status"), "usageReason": usage.get("reason"), - }) - rows.sort(key=lambda item: (item.get("score") is None, -(item.get("score") or -1), int(item.get("accountId") or 0))) - return { - "source": "sub2api-native-admin-usage-request-errors-and-system-logs", - "scope": "group-account-window", "window": PAYLOAD.get("since"), - "scoreRule": { - "reliabilityWeight": 60, "latencyWeight": 25, "availabilityWeight": 15, - "failureDefinition": "unique request ids with account failover, forward_failed, or customer-visible error", - "reliability": "linear from 60 points at 0% observed failures to 0 points at 20%", - "latency": "requires at least 5 first-token samples; linear from 25 points at <=10s TTFT P95 to 0 points at >=180s", - "availability": "15 points when active, schedulable, and currently available; 8 when active but unavailable; otherwise 0", - "missingMetric": "missing latency is excluded from the denominator and lowers confidence; it is not treated as zero latency", - }, - "accounts": rows, - "attribution": "Success denominators and first-token latency come from native admin usage records. Failures are deduplicated by request id across native request-errors and indexed failover/forward events. Scores are analytical and never mutate scheduling.", - } - -def all_group_error_overview(token, groups): - page_size = 20 - cursor = int(PAYLOAD.get("pageToken") or 0) - ordered = sorted( - [item for item in groups if isinstance(item, dict) and isinstance(item.get("id"), int)], - key=lambda item: item["id"], - ) - remaining = [item for item in ordered if item["id"] > cursor] - page = remaining[:page_size] - summaries = [] - totals = { - "requestCount": 0, - "errorCount": 0, - "businessLimitedCount": 0, - "upstreamErrorCount": 0, - "groupsNeedingAttention": 0, - } - for group in page: - group_id = group["id"] - platform = PAYLOAD.get("platform") or group.get("platform") - snapshot = native_ops_snapshot(token, group_id, platform) - overview = ((snapshot.get("overview") or {}).get("summary") or {}) - availability = ((snapshot.get("accountAvailability") or {}).get("summary") or {}) - availability_group = availability.get("group") if isinstance(availability.get("group"), dict) else {} - concurrency = ((snapshot.get("concurrency") or {}).get("summary") or {}) - concurrency_group = concurrency.get("group") if isinstance(concurrency.get("group"), dict) else {} - request_count = overview.get("request_count_total") - error_count = overview.get("error_count_total") - business_limited = overview.get("business_limited_count") - upstream_error_count = sum( - value for value in ( - overview.get("upstream_error_count_excl_429_529"), - overview.get("upstream_429_count"), - overview.get("upstream_529_count"), - ) if isinstance(value, int) - ) - unavailable_count = availability.get("unavailableAccountCount") - needs_attention = ( - (isinstance(error_count, int) and error_count > 0) - or upstream_error_count > 0 - or (isinstance(unavailable_count, int) and unavailable_count > 0) - ) - totals["requestCount"] += request_count if isinstance(request_count, int) else 0 - totals["errorCount"] += error_count if isinstance(error_count, int) else 0 - totals["businessLimitedCount"] += business_limited if isinstance(business_limited, int) else 0 - totals["upstreamErrorCount"] += upstream_error_count - totals["groupsNeedingAttention"] += 1 if needs_attention else 0 - summaries.append({ - "groupId": group_id, - "groupName": group.get("name"), - "platform": platform, - "status": group.get("status"), - "requestCount": request_count, - "errorCount": error_count, - "errorRate": overview.get("error_rate"), - "upstreamErrorCount": upstream_error_count, - "upstreamErrorRate": overview.get("upstream_error_rate"), - "businessLimitedCount": business_limited, - "ttftP99Ms": ((overview.get("ttft") or {}).get("p99_ms") if isinstance(overview.get("ttft"), dict) else None), - "durationP99Ms": ((overview.get("duration") or {}).get("p99_ms") if isinstance(overview.get("duration"), dict) else None), - "totalAccounts": availability_group.get("total_accounts"), - "availableCount": availability_group.get("available_count"), - "rateLimitCount": availability_group.get("rate_limit_count"), - "errorAccountCount": availability_group.get("error_count"), - "unavailableAccountCount": unavailable_count, - "currentInUse": concurrency_group.get("current_in_use"), - "maxCapacity": concurrency_group.get("max_capacity"), - "waitingInQueue": concurrency_group.get("waiting_in_queue"), - "needsAttention": needs_attention, - "opsStatus": { - "overview": (snapshot.get("overview") or {}).get("status"), - "accountAvailability": (snapshot.get("accountAvailability") or {}).get("status"), - "concurrency": (snapshot.get("concurrency") or {}).get("status"), - }, - }) - next_token = str(page[-1]["id"]) if len(remaining) > page_size and page else None - totals["errorRate"] = round(totals["errorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None - totals["upstreamErrorRate"] = round(totals["upstreamErrorCount"] / totals["requestCount"], 6) if totals["requestCount"] > 0 else None - ttft_p99_values = [item.get("ttftP99Ms") for item in summaries if isinstance(item.get("ttftP99Ms"), int)] - max_ttft_p99 = max(ttft_p99_values) if ttft_p99_values else None - observed_floor = max(60, ((max_ttft_p99 + 29999) // 15000) * 15) if isinstance(max_ttft_p99, int) else None - return { - "scope": "all-groups-overview", - "window": {"since": PAYLOAD.get("since")}, - "platform": PAYLOAD.get("platform"), - "totalGroups": len(ordered), - "returnedGroups": len(summaries), - "pageSize": page_size, - "pageToken": str(cursor) if cursor > 0 else None, - "nextPageToken": next_token, - "pageTotals": totals, - "responseHeaderTimeout": runtime_response_header_timeout(), - "requestTiming": { - "maxTtftP99Ms": max_ttft_p99, - "observedFloorSeconds": observed_floor, - "scope": "returned-groups-page", - "attribution": "The observed floor is the maximum returned-group TTFT P99 plus 15 seconds rounded up to 15 seconds. It is analytical only and must not be treated as a safe timeout.", - }, - "groups": summaries, - "disclosure": "Page totals cover only returned groups. Follow nextPageToken for remaining groups and use --group for account root causes and request indexes.", - } - -def native_system_log_lines(token, platform=None): - start_time = quote(since_start_time(PAYLOAD.get("since")), safe="") - platform_query = "&platform=" + quote(str(platform), safe="") if platform else "" - markers = ( - "account_temp_unschedulable", "openai.upstream_failover_switching", - "openai.account_select_failed", "openai.forward_failed", - "openai.pool_mode_same_account_retry", "gateway.failover_same_account_retry", - "http request completed", - ) - rows_by_id = {} - queries = [] - for marker in markers: - page = 1 - returned = 0 - total = None - try: - while True: - path = f"/api/v1/admin/ops/system-logs?page={page}&page_size=200&start_time={start_time}&q={quote(marker, safe='')}{platform_query}" - data = data_of(curl_api("GET", path, bearer=token), "list ops system logs") - rows = items_of(data) - returned += len(rows) - if total is None and isinstance(data, dict) and isinstance(data.get("total"), int): - total = data.get("total") - for row in rows: - if not isinstance(row, dict): - continue - row_id = row.get("id") - key = str(row_id) if row_id is not None else json.dumps(row, sort_keys=True, default=str) - rows_by_id[key] = row - if not rows or len(rows) < 200 or (isinstance(total, int) and returned >= total): - break - page += 1 - except Exception as exc: - return None, { - "status": "unavailable", - "source": "sub2api-native-admin-ops-system-logs", - "reason": compact_error(str(exc)), - "fallback": "runtime-container-logs", - } - queries.append({"marker": marker, "returned": returned, "total": total if total is not None else returned, "pagesRead": page, "truncated": False}) - lines = [] - for row in sorted(rows_by_id.values(), key=lambda item: str(item.get("created_at") or "")): - extra = dict(row.get("extra") or {}) if isinstance(row.get("extra"), dict) else {} - for source_key, target_key in ( - ("request_id", "request_id"), ("client_request_id", "client_request_id"), - ("account_id", "account_id"), ("platform", "platform"), ("model", "model"), - ): - if extra.get(target_key) is None and row.get(source_key) is not None: - extra[target_key] = row.get(source_key) - message = str(row.get("message") or "") - lines.append(str(row.get("created_at") or "") + "\\t" + message + "\\t" + json.dumps(extra, separators=(",", ":"), default=str)) - return lines, { - "status": "available", - "source": "sub2api-native-admin-ops-system-logs", - "uniqueEventCount": len(lines), - "queries": queries, - "fallbackUsed": False, - } - -def policy_effectiveness(lines, accounts, source, group_id=None): - account_names = {str(item.get("id")): item.get("name") for item in accounts} - account_ids = set(account_names.keys()) - account_policy = { - key: { - "accountId": item.get("id"), "accountName": item.get("name"), - "tempUnschedulableEvents": 0, "failoverEvents": 0, "failoverRequestIds": set(), - "failoverRecoveredRequestIds": set(), "failoverFailedRequestIds": set(), "failoverOutcomeMissingRequestIds": set(), - "forwardFailedEvents": 0, "forwardFailedRequestIds": set(), "sameAccountRetryEvents": 0, - "upstreamStatusCodes": {}, - } - for key, item in ((str(item.get("id")), item) for item in accounts) - } - requests = {} - counts = { - "tempUnschedulableEvents": 0, - "failoverEvents": 0, - "selectFailedEvents": 0, - "forwardFailedEvents": 0, - "sameAccountRetryEvents": 0, - "gatewayCompletedEvents": 0, - "gatewayFinalErrorEvents": 0, - } - missing_request_ids = { - "tempUnschedulable": 0, - "failover": 0, - "selectFailed": 0, - "forwardFailed": 0, - "sameAccountRetry": 0, - "gatewayCompleted": 0, - } - temp_accounts = {} - temp_samples = [] - failover_samples = [] - select_failed_samples = [] - final_statuses = {} - event_line_count = 0 - - def request_entry(request_id): - if not isinstance(request_id, str) or not request_id: - return None - if request_id not in requests: - requests[request_id] = { - "requestId": request_id, - "tempUnschedulableCount": 0, - "failoverCount": 0, - "selectFailedCount": 0, - "forwardFailedCount": 0, - "sameAccountRetryCount": 0, - "finalStatus": None, - } - return requests[request_id] - - for line in lines: - if not any(marker in line for marker in ( - "account_temp_unschedulable", "openai.upstream_failover_switching", - "openai.account_select_failed", "openai.forward_failed", - "openai.pool_mode_same_account_retry", "gateway.failover_same_account_retry", - "http request completed", - )): - continue - json_start = line.find("{") - if json_start < 0: - continue - try: - item = json.loads(line[json_start:]) - except json.JSONDecodeError: - continue - if not isinstance(item, dict): - continue - at = line.split("\\t", 1)[0].strip() or None - request_id = item.get("request_id") - account_id = item.get("account_id") - event_group_id = item.get("group_id") - if event_group_id is not None and group_id is not None and str(event_group_id) != str(group_id): - continue - if account_id is not None and str(account_id) not in account_ids: - continue - explicitly_scoped = account_id is not None or event_group_id is not None - if not explicitly_scoped and (not isinstance(request_id, str) or request_id not in requests): - continue - request = request_entry(request_id) - base = { - "at": at, - "requestId": request_id, - "accountId": account_id, - "accountName": account_names.get(str(account_id)), - } - raw_account_stat = account_policy.get(str(account_id)) - account_stat = raw_account_stat if group_id is None or event_group_id is not None else None - if "account_temp_unschedulable" in line: - event_line_count += 1 - counts["tempUnschedulableEvents"] += 1 - if request is not None: - request["tempUnschedulableCount"] += 1 - else: - missing_request_ids["tempUnschedulable"] += 1 - key = str(account_id) if account_id is not None else "unknown" - if key not in temp_accounts: - temp_accounts[key] = { - "accountId": account_id, - "accountName": account_names.get(str(account_id)), - "count": 0, - } - temp_accounts[key]["count"] += 1 - if raw_account_stat is not None: - raw_account_stat["tempUnschedulableEvents"] += 1 - if len(temp_samples) < 20: - temp_samples.append({ - **base, - "statusCode": item.get("status_code"), - "ruleIndex": item.get("rule_index"), - "matchedKeyword": item.get("matched_keyword"), - "until": item.get("until") or item.get("temp_unschedulable_until"), - "reason": compact_error(item.get("reason") or item.get("error")), - }) - elif "openai.upstream_failover_switching" in line: - event_line_count += 1 - counts["failoverEvents"] += 1 - if request is not None: - request["failoverCount"] += 1 - else: - missing_request_ids["failover"] += 1 - if account_stat is not None: - account_stat["failoverEvents"] += 1 - if isinstance(request_id, str) and request_id: - account_stat["failoverRequestIds"].add(request_id) - status_code = item.get("upstream_status") - if isinstance(status_code, int): - key = str(status_code) - account_stat["upstreamStatusCodes"][key] = account_stat["upstreamStatusCodes"].get(key, 0) + 1 - if len(failover_samples) < 20: - failover_samples.append({ - **base, - "switchCount": item.get("switch_count"), - "maxSwitches": item.get("max_switches"), - }) - elif "openai.account_select_failed" in line: - event_line_count += 1 - counts["selectFailedEvents"] += 1 - if request is not None: - request["selectFailedCount"] += 1 - else: - missing_request_ids["selectFailed"] += 1 - if len(select_failed_samples) < 20: - select_failed_samples.append({ - **base, - "excludedAccountCount": item.get("excluded_account_count"), - "error": compact_error(item.get("error")), - }) - elif "openai.forward_failed" in line: - event_line_count += 1 - counts["forwardFailedEvents"] += 1 - if request is not None: - request["forwardFailedCount"] += 1 - else: - missing_request_ids["forwardFailed"] += 1 - if account_stat is not None: - account_stat["forwardFailedEvents"] += 1 - if isinstance(request_id, str) and request_id: - account_stat["forwardFailedRequestIds"].add(request_id) - elif "openai.pool_mode_same_account_retry" in line or "gateway.failover_same_account_retry" in line: - event_line_count += 1 - counts["sameAccountRetryEvents"] += 1 - if request is not None: - request["sameAccountRetryCount"] += 1 - else: - missing_request_ids["sameAccountRetry"] += 1 - if raw_account_stat is not None: - raw_account_stat["sameAccountRetryEvents"] += 1 - elif "http request completed" in line and gateway_request_path(item.get("path")): - event_line_count += 1 - counts["gatewayCompletedEvents"] += 1 - status_code = item.get("status_code") - if request is not None and isinstance(status_code, int): - request["finalStatus"] = status_code - elif request is None: - missing_request_ids["gatewayCompleted"] += 1 - if isinstance(status_code, int): - key = str(status_code) - final_statuses[key] = final_statuses.get(key, 0) + 1 - if status_code >= 400: - counts["gatewayFinalErrorEvents"] += 1 - - correlated = [item for item in requests.values() if item["failoverCount"] > 0] - succeeded = [item for item in correlated if isinstance(item.get("finalStatus"), int) and item["finalStatus"] < 400] - failed = [item for item in correlated if isinstance(item.get("finalStatus"), int) and item["finalStatus"] >= 400] - incomplete = [item for item in correlated if not isinstance(item.get("finalStatus"), int)] - cooled_then_failover = [item for item in correlated if item["tempUnschedulableCount"] > 0] - forward_with_http_success = [ - item for item in requests.values() - if item["forwardFailedCount"] > 0 and isinstance(item.get("finalStatus"), int) and item["finalStatus"] < 400 - ] - forward_with_http_error = [ - item for item in requests.values() - if item["forwardFailedCount"] > 0 and isinstance(item.get("finalStatus"), int) and item["finalStatus"] >= 400 - ] - forward_outcome_missing = [ - item for item in requests.values() - if item["forwardFailedCount"] > 0 and not isinstance(item.get("finalStatus"), int) - ] - for request in correlated: - request_id = request.get("requestId") - if not isinstance(request_id, str): - continue - for current in account_policy.values(): - if request_id not in current["failoverRequestIds"]: - continue - final_status = request.get("finalStatus") - if isinstance(final_status, int) and final_status < 400: - current["failoverRecoveredRequestIds"].add(request_id) - elif isinstance(final_status, int): - current["failoverFailedRequestIds"].add(request_id) - else: - current["failoverOutcomeMissingRequestIds"].add(request_id) - account_policy_rows = [] - for current in account_policy.values(): - account_policy_rows.append({ - "accountId": current["accountId"], "accountName": current["accountName"], - "tempUnschedulableEvents": current["tempUnschedulableEvents"], - "failoverEvents": current["failoverEvents"], "failoverRequests": len(current["failoverRequestIds"]), - "failoverRecovered": len(current["failoverRecoveredRequestIds"]), - "failoverFailed": len(current["failoverFailedRequestIds"]), - "failoverOutcomeMissing": len(current["failoverOutcomeMissingRequestIds"]), - "forwardFailedEvents": current["forwardFailedEvents"], "forwardFailedRequests": len(current["forwardFailedRequestIds"]), - "sameAccountRetryEvents": current["sameAccountRetryEvents"], - "_failureRequestIds": sorted(current["failoverRequestIds"] | current["forwardFailedRequestIds"]), - "upstreamStatusBuckets": [ - {"statusCode": int(code), "count": count} - for code, count in sorted(current["upstreamStatusCodes"].items(), key=lambda pair: int(pair[0])) - ], - }) - sample_limit = 20 if PAYLOAD.get("full") is True else (2 if PAYLOAD.get("account") is not None else 5) - completed = counts["gatewayCompletedEvents"] - final_errors = counts["gatewayFinalErrorEvents"] - result = { - "source": source, - "scope": "group-membership-window", - "attribution": "Events with group_id are exact. Events without group_id are attributed by account membership and request_id correlation; shared-account events can appear in more than one group drilldown.", - "eventLineCount": event_line_count, - **counts, - "tempUnschedulableAccounts": sorted(temp_accounts.values(), key=lambda item: (-item["count"], str(item.get("accountName") or ""))), - "accountPolicyMetrics": sorted(account_policy_rows, key=lambda item: int(item.get("accountId") or 0)), - "accountPolicyAttribution": "Failover and forward-failure request ids enter group scoring only when system-log events carry group_id. Account temp-unschedulable and same-account retry counts are account-wide operational signals and are not added to the failure-rate numerator.", - "failoverRequestCount": len(correlated), - "failoverSucceededRequestCount": len(succeeded), - "failoverFailedRequestCount": len(failed), - "failoverOutcomeMissingRequestCount": len(incomplete), - "cooledThenFailoverRequestCount": len(cooled_then_failover), - "forwardFailureWithHttpSuccessRequestCount": len(forward_with_http_success), - "forwardFailureWithHttpErrorRequestCount": len(forward_with_http_error), - "forwardFailureOutcomeMissingRequestCount": len(forward_outcome_missing), - "requestIdCoverage": { - "missingByEventType": missing_request_ids, - "tempToFailoverCorrelationAvailable": counts["tempUnschedulableEvents"] > missing_request_ids["tempUnschedulable"], - }, - "gatewayFinalStatusBuckets": [ - {"statusCode": int(code), "count": count} - for code, count in sorted(final_statuses.items(), key=lambda pair: int(pair[0])) - ], - "gatewayOutcomeEvidence": { - "completedLogEventCount": completed, - "finalErrorLogEventCount": final_errors, - "observedFinalErrorShare": round(final_errors / completed, 4) if completed > 0 else None, - "denominator": "gateway http request completed log events for Responses, Messages, and Chat Completions paths", - "coverage": "since-window-and-tail-bounded", - "completeTrafficDenominatorGuaranteed": False, - }, - "requestDrilldown": { - "succeededAfterFailover": [item["requestId"] for item in succeeded[:sample_limit]], - "failedAfterFailover": [item["requestId"] for item in failed[:sample_limit]], - "outcomeMissingAfterFailover": [item["requestId"] for item in incomplete[:sample_limit]], - "selectFailed": [item["requestId"] for item in requests.values() if item["selectFailedCount"] > 0][:sample_limit], - "forwardFailureWithHttpSuccess": [item["requestId"] for item in forward_with_http_success[:sample_limit]], - "forwardFailureWithHttpError": [item["requestId"] for item in forward_with_http_error[:sample_limit]], - "forwardFailureOutcomeMissing": [item["requestId"] for item in forward_outcome_missing[:sample_limit]], - }, - "assessment": ( - "failover-observed-with-success" if succeeded else - "failover-observed-without-success" if correlated else - "no-failover-observed" - ), - "disclosure": "Rule matches require account_temp_unschedulable evidence; status codes alone are not treated as rule hits. Request outcomes are joined only by request_id. HTTP success can coexist with forward_failed on streaming responses and does not prove the client received a complete terminal event.", - } - if PAYLOAD.get("full") is True: - result["samples"] = { - "tempUnschedulable": temp_samples, - "failovers": failover_samples, - "selectFailures": select_failed_samples, - } - return result - -def observed_runtime_errors(token, accounts, group_id, platform=None): - selector = PAYLOAD.get("account") - selected = [item for item in accounts if selector is None or str(item.get("id")) == str(selector) or item.get("name") == selector] - if selector is not None and len(selected) != 1: - raise RuntimeError("account-not-found-or-ambiguous: " + str(selector)) - selected_ids = set(str(item.get("id")) for item in selected) - stats = {} - for item in selected: - stats[str(item.get("id"))] = { - "accountName": item.get("name"), - "accountId": item.get("id"), - "upstreamEventCount": 0, - "observedStatusCodes": {}, - "forwardFailureCount": 0, - "forwardFailureStatusCodes": {}, - "lastSeenAt": None, - "errorSamples": [], - } - native_ops = native_ops_snapshot(token, group_id, platform) - lines, system_log_source = native_system_log_lines(token, platform) - if lines is None: - lines = runtime_log_lines() - system_log_source["fallbackUsed"] = True - else: - marker_queries = [item for item in system_log_source.get("queries", []) if item.get("marker") != "http request completed"] - marker_total = sum(item.get("total", 0) for item in marker_queries if isinstance(item.get("total"), int)) - overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) - upstream_count = overview_summary.get("upstream_error_count_excl_429_529") or 0 - upstream_count += overview_summary.get("upstream_429_count") or 0 - upstream_count += overview_summary.get("upstream_529_count") or 0 - if marker_total == 0 and upstream_count > 0: - runtime_event_lines = [ - line for line in runtime_log_lines() - if any(marker in line for marker in ( - "account_temp_unschedulable", "openai.upstream_failover_switching", - "openai.account_select_failed", "openai.forward_failed", - )) - ] - lines.extend(runtime_event_lines) - system_log_source["source"] = "sub2api-native-admin-ops-with-runtime-event-fallback" - system_log_source["fallbackUsed"] = True - system_log_source["fallbackReason"] = "native system-log index returned no policy events while the native overview reported upstream errors" - system_log_source["fallbackEventCount"] = len(runtime_event_lines) - effectiveness = policy_effectiveness(lines, selected if selector is not None else accounts, system_log_source.get("source"), group_id) - effectiveness["sourceStatus"] = system_log_source - overview_summary = ((native_ops.get("overview") or {}).get("summary") or {}) - native_ops["responseHeaderTimeout"] = runtime_response_header_timeout() - native_ops["requestTiming"] = native_request_timing(token, group_id, platform, overview_summary) - native_ops["customerErrors"] = native_customer_error_profile(token, group_id, platform) - native_ops["accountQuality"] = account_quality_scores(token, selected, group_id, native_ops, effectiveness) - request_total = overview_summary.get("request_count_total") - error_total = overview_summary.get("error_count_total") - if isinstance(request_total, int) and isinstance(error_total, int): - effectiveness["gatewayOutcomeEvidence"] = { - "source": "sub2api-native-admin-ops-dashboard-overview", - "requestCountTotal": request_total, - "errorCountTotal": error_total, - "errorRate": overview_summary.get("error_rate"), - "upstreamErrorRate": overview_summary.get("upstream_error_rate"), - "businessLimitedCount": overview_summary.get("business_limited_count"), - "startTime": overview_summary.get("start_time"), - "endTime": overview_summary.get("end_time"), - "completeTrafficDenominatorGuaranteed": True, - } - scanned = 0 - for line in lines: - if "account_upstream_error" not in line and "openai.forward_failed" not in line: - continue - json_start = line.find("{") - if json_start < 0: - continue - try: - item = json.loads(line[json_start:]) - except json.JSONDecodeError: - continue - account_id = item.get("account_id") if isinstance(item, dict) else None - account_key = str(account_id) - if account_key not in selected_ids: - continue - scanned += 1 - at = line.split("\t", 1)[0].strip() or None - current = stats[account_key] - if at and (current["lastSeenAt"] is None or at > current["lastSeenAt"]): - current["lastSeenAt"] = at - if "account_upstream_error" in line: - code = item.get("status_code") - if isinstance(code, int): - key = str(code) - current["observedStatusCodes"][key] = current["observedStatusCodes"].get(key, 0) + 1 - current["upstreamEventCount"] += 1 - continue - current["forwardFailureCount"] += 1 - error = compact_error(item.get("error")) - match = re.search(r"(?:upstream error:\\s*|status(?:_code)?[=:]\\s*)(\\d{3})", error, re.IGNORECASE) - if match: - key = match.group(1) - current["forwardFailureStatusCodes"][key] = current["forwardFailureStatusCodes"].get(key, 0) + 1 - if error and error not in current["errorSamples"] and len(current["errorSamples"]) < 3: - current["errorSamples"].append(error) - rows = [] - aggregate = {} - for item in selected: - current = stats[str(item.get("id"))] - observed = [{"statusCode": int(code), "count": count} for code, count in sorted(current["observedStatusCodes"].items(), key=lambda pair: int(pair[0]))] - forwarded = [{"statusCode": int(code), "count": count} for code, count in sorted(current["forwardFailureStatusCodes"].items(), key=lambda pair: int(pair[0]))] - for entry in observed: - key = str(entry["statusCode"]) - aggregate[key] = aggregate.get(key, 0) + entry["count"] - current["observedStatusCodes"] = observed - current["forwardFailureStatusCodes"] = forwarded - analysis = ops_error_analysis(token, item, observed) - has_ops_errors = analysis.get("status") == "available" and (analysis.get("opsRecordCount") or 0) > 0 - if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or has_ops_errors: - if PAYLOAD.get("full") is not True and analysis.get("status") == "available": - analysis = { - "source": analysis.get("source"), - "status": analysis.get("status"), - "opsRecordCount": analysis.get("opsRecordCount"), - "recordsTruncated": analysis.get("recordsTruncated"), - "statusBuckets": analysis.get("statusBuckets"), - "causeBuckets": analysis.get("causeBuckets"), - "rootCause": analysis.get("rootCause"), - "errorRateEvidence": analysis.get("errorRateEvidence"), - "concurrencyRecommendation": analysis.get("concurrencyRecommendation"), - "runtimeConcurrency": analysis.get("runtimeConcurrency"), - "samples": (analysis.get("samples") or [])[:1], - } - current["opsAnalysis"] = analysis - if PAYLOAD.get("full") is not True: - current["errorSamples"] = current["errorSamples"][:1] - if current["upstreamEventCount"] > 0 or current["forwardFailureCount"] > 0 or has_ops_errors or selector is not None: - rows.append(current) - root_causes = {} - account_roots = [] - for row in rows: - analysis = row.get("opsAnalysis") - if not isinstance(analysis, dict) or analysis.get("status") != "available": - continue - for bucket in analysis.get("causeBuckets") or []: - cause = bucket.get("cause") - count = bucket.get("count") - if isinstance(cause, str) and isinstance(count, int): - root_causes[cause] = root_causes.get(cause, 0) + count - root = analysis.get("rootCause") or {} - account_roots.append({ - "accountId": row.get("accountId"), - "accountName": row.get("accountName"), - "dominantCause": root.get("dominantCause"), - "dominantCount": root.get("dominantCount"), - "dominantShare": root.get("dominantShare"), - }) - ranked_causes = sorted(root_causes.items(), key=lambda pair: (-pair[1], pair[0])) - native_overview = native_ops.get("overview") or {} - native_overview_values = native_overview.get("summary") if isinstance(native_overview.get("summary"), dict) else native_overview - native_error_rate_available = isinstance(native_overview_values.get("error_rate") if isinstance(native_overview_values, dict) else None, (int, float)) - account_count_with_errors = sum( - 1 for item in rows - if item["upstreamEventCount"] > 0 - or item["forwardFailureCount"] > 0 - or (((item.get("opsAnalysis") or {}).get("opsRecordCount") or 0) > 0) - ) - if selector is None and PAYLOAD.get("full") is not True: - for row in rows: - row.pop("opsAnalysis", None) - if PAYLOAD.get("full") is not True: - overview = ((native_ops.get("overview") or {}).get("summary") or {}) - availability = ((native_ops.get("accountAvailability") or {}).get("summary") or {}) - concurrency = ((native_ops.get("concurrency") or {}).get("summary") or {}) - sink = ((native_ops.get("systemLogSink") or {}).get("health") or {}) - unavailable_accounts = availability.get("unavailableAccounts") or [] - if selector is not None: - unavailable_accounts = [item for item in unavailable_accounts if item.get("accountId") in selected_ids] - native_ops = { - "source": native_ops.get("source"), - "overview": { - "status": (native_ops.get("overview") or {}).get("status"), - "healthScore": overview.get("health_score"), - "requestCount": overview.get("request_count_total"), - "errorCount": overview.get("error_count_total"), - "errorRate": overview.get("error_rate"), - "upstreamErrorRate": overview.get("upstream_error_rate"), - "businessLimitedCount": overview.get("business_limited_count"), - }, - "responseHeaderTimeout": native_ops.get("responseHeaderTimeout"), - "requestTiming": native_ops.get("requestTiming"), - "customerErrors": native_ops.get("customerErrors"), - "accountQuality": native_ops.get("accountQuality"), - "accountAvailability": { - "status": (native_ops.get("accountAvailability") or {}).get("status"), - "group": availability.get("group"), - "unavailableAccountCount": availability.get("unavailableAccountCount"), - "unavailableAccounts": unavailable_accounts, - }, - "concurrency": { - "status": (native_ops.get("concurrency") or {}).get("status"), - "group": concurrency.get("group"), - }, - "systemLogSink": { - "status": (native_ops.get("systemLogSink") or {}).get("status"), - "queueDepth": sink.get("queue_depth"), - "droppedCount": sink.get("dropped_count"), - "writeFailedCount": sink.get("write_failed_count"), - }, - } - source_status = effectiveness.get("sourceStatus") or {} - compact_source_status = { - "status": source_status.get("status"), - "source": source_status.get("source"), - "fallbackUsed": source_status.get("fallbackUsed"), - "fallbackReason": source_status.get("fallbackReason"), - "fallbackEventCount": source_status.get("fallbackEventCount"), - "nativeIndexedEventCounts": { - item.get("marker"): item.get("total") - for item in source_status.get("queries", []) - if isinstance(item, dict) - }, - } - drilldown = effectiveness.get("requestDrilldown") or {} - effectiveness = { - "source": effectiveness.get("source"), - "scope": effectiveness.get("scope"), - "attribution": effectiveness.get("attribution"), - "assessment": effectiveness.get("assessment"), - "ruleMatches": { - "tempUnschedulableEvents": effectiveness.get("tempUnschedulableEvents"), - "accounts": effectiveness.get("tempUnschedulableAccounts"), - }, - "failover": { - "events": effectiveness.get("failoverEvents"), - "requests": effectiveness.get("failoverRequestCount"), - "succeeded": effectiveness.get("failoverSucceededRequestCount"), - "failed": effectiveness.get("failoverFailedRequestCount"), - "outcomeMissing": effectiveness.get("failoverOutcomeMissingRequestCount"), - }, - "forwardFailure": { - "events": effectiveness.get("forwardFailedEvents"), - "withHttpSuccess": effectiveness.get("forwardFailureWithHttpSuccessRequestCount"), - "withHttpError": effectiveness.get("forwardFailureWithHttpErrorRequestCount"), - "outcomeMissing": effectiveness.get("forwardFailureOutcomeMissingRequestCount"), - }, - "selectFailedEvents": effectiveness.get("selectFailedEvents"), - "requestIdCoverage": effectiveness.get("requestIdCoverage"), - "gatewayOutcomeEvidence": effectiveness.get("gatewayOutcomeEvidence"), - "requestDrilldown": {key: value for key, value in drilldown.items() if value}, - "sourceStatus": compact_source_status, - "disclosure": effectiveness.get("disclosure"), - } - if selector is not None: - native_ops["accountAvailability"].pop("group", None) - native_ops.pop("concurrency", None) - native_ops.pop("systemLogSink", None) - effectiveness["sourceStatus"] = { - "source": compact_source_status.get("source"), - "fallbackUsed": compact_source_status.get("fallbackUsed"), - "fallbackReason": compact_source_status.get("fallbackReason"), - } - return { - "window": {"since": PAYLOAD.get("since"), "tail": PAYLOAD.get("tail"), "matchedLogEvents": scanned}, - "aggregateObservedStatusCodes": [{"statusCode": int(code), "count": count} for code, count in sorted(aggregate.items(), key=lambda pair: int(pair[0]))], - "accountCountWithErrors": account_count_with_errors, - "rootCauseAnalysis": { - "source": "ops-upstream-errors", - "scope": "account-window-for-group-members", - "attribution": "The upstream-errors endpoint is account scoped. If accounts belong to multiple groups, these cause buckets are shared evidence and must not be added across groups.", - "causeBuckets": [{"cause": cause, "count": count} for cause, count in ranked_causes], - "dominantCause": ranked_causes[0][0] if ranked_causes else None, - "dominantCount": ranked_causes[0][1] if ranked_causes else 0, - "accountRoots": account_roots, - "errorRateAvailable": native_error_rate_available, - }, - "nativeOps": native_ops, - "policyEffectiveness": effectiveness, - **( - {"accounts": rows} - if selector is not None or PAYLOAD.get("full") is True - else {"accountDrilldown": "rerun runtime errors with --account for per-account evidence"} - ), - "message": "observedStatusCodes comes only from account_upstream_error; forward failures are supplemental message evidence", - } - -def selected_account_details(token, accounts): - selectors = PAYLOAD.get("accounts") - if not isinstance(selectors, list) or not selectors: - selector = PAYLOAD.get("account") - selectors = [selector] if isinstance(selector, str) and selector else [] - details = [] - selected_ids = set() - for raw_selector in selectors: - selector = str(raw_selector) - detail = find_account(token, accounts, selector) - account_id = detail.get("id") - if account_id in selected_ids: - raise RuntimeError("duplicate-account-selector: " + selector) - if PAYLOAD.get("kind") == "temp-unschedulable" and detail.get("type") != "apikey": - raise RuntimeError("runtime temp-unschedulable policy requires an apikey account: " + selector) - selected_ids.add(account_id) - details.append(detail) - return details - -def mutation_plan(detail, action, include_rules): - if PAYLOAD.get("kind") == "priority": - before_priority = int(detail.get("priority") or 0) - desired_priority = int(PAYLOAD.get("priority")) - return { - "accountName": detail.get("name"), - "accountId": detail.get("id"), - "change": "noop" if before_priority == desired_priority else "update", - "before": {"priority": before_priority}, - "desired": {"priority": desired_priority}, - "template": None, - "_kind": "priority", - "_desiredPriority": desired_priority, - } - credentials = dict(detail.get("credentials") or {}) - before = normalize_policy(credentials) - if action == "apply": - template = PAYLOAD.get("selectedTemplate") - if not isinstance(template, dict): - raise RuntimeError("selected runtime template missing") - desired = dict(credentials) - desired.update(template.get("credentials") or {}) - desired_policy = normalize_policy(desired) - exists = "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials - change = "noop" if before == desired_policy else ("update" if exists else "create") - else: - desired = dict(credentials) - desired.pop("temp_unschedulable_enabled", None) - desired.pop("temp_unschedulable_rules", None) - desired_policy = normalize_policy(desired) - change = "delete" if before != desired_policy or "temp_unschedulable_enabled" in credentials or "temp_unschedulable_rules" in credentials else "noop" - return { - "accountName": detail.get("name"), - "accountId": detail.get("id"), - "change": change, - "before": policy_summary(before, include_rules), - "desired": policy_summary(desired_policy, include_rules), - "template": PAYLOAD.get("selectedTemplate", {}).get("id") if isinstance(PAYLOAD.get("selectedTemplate"), dict) else None, - "_desiredCredentials": desired, - "_desiredPolicy": desired_policy, - "_kind": "temp-unschedulable", - } - -def write_mutation_plan(token, plan): - if plan["change"] == "noop": - return {"attempted": False, "status": "noop", "error": None} - try: - payload = {"priority": plan["_desiredPriority"]} if plan["_kind"] == "priority" else {"credentials": plan["_desiredCredentials"]} - data_of(curl_api("PUT", f"/api/v1/admin/accounts/{plan['accountId']}", bearer=token, payload=payload), "update account runtime config") - return {"attempted": True, "status": "succeeded", "error": None} - except Exception as exc: - return {"attempted": True, "status": "failed", "error": compact_error(str(exc))} - -def reconcile_mutation_plan(token, plan, write, include_rules): - item = {key: value for key, value in plan.items() if not key.startswith("_")} - item["write"] = write - try: - actual_detail = account_detail(token, plan["accountId"]) - if plan["_kind"] == "priority": - actual_priority = int(actual_detail.get("priority") or 0) - item["actual"] = {"priority": actual_priority} - item["reconciled"] = actual_priority == plan["_desiredPriority"] - else: - actual_policy = normalize_policy(actual_detail.get("credentials")) - item["actual"] = policy_summary(actual_policy, include_rules) - item["reconciled"] = actual_policy == plan["_desiredPolicy"] - item["reconcileError"] = None - except Exception as exc: - item["actual"] = None - item["reconciled"] = False - item["reconcileError"] = compact_error(str(exc)) - return item - -def runtime_result(): - token = login() - action = PAYLOAD["action"] - if action == "errors" and PAYLOAD.get("allGroups") is True: - groups = list_groups(token, PAYLOAD.get("platform")) - overview = all_group_error_overview(token, groups) - return { - "ok": True, - "operation": "errors", - "mutation": False, - "endpoint": "host-docker" if RUNTIME_MODE == "host-docker" else "k3s-service", - "allGroups": overview, - "valuesPrinted": False, - } - group_selector = PAYLOAD.get("group") - group_platform = PAYLOAD.get("platform") if action in ("errors", "infrastructure") else "openai" - if action in ("errors", "infrastructure") and group_selector is None: - group_platform = "openai" - groups = list_groups(token, group_platform) - selector = str(group_selector if group_selector is not None else PAYLOAD["groupName"]) - matching_groups = [item for item in groups if str(item.get("id")) == selector or item.get("name") == selector] - if len(matching_groups) > 1: - raise RuntimeError("group-selector-ambiguous: " + selector) - group = matching_groups[0] if matching_groups else None - if not isinstance(group, dict) or group.get("id") is None: - raise RuntimeError("group-not-found: " + selector) - effective_platform = group_platform or group.get("platform") - accounts = list_group_accounts(token, group["id"], effective_platform) - base = { - "group": {"id": group.get("id"), "name": group.get("name"), "platform": effective_platform, "status": group.get("status")}, - "templates": [template_summary(item) for item in PAYLOAD["templates"]], - "endpoint": "host-docker" if RUNTIME_MODE == "host-docker" else "k3s-service", - "valuesPrinted": False, - } - if action == "list": - details = [account_detail(token, item["id"]) for item in accounts] - return {**base, "ok": True, "operation": "list", "mutation": False, "accountCount": len(details), "accounts": [account_summary(item) for item in sorted(details, key=lambda value: value.get("name") or "")]} - if action == "errors": - errors = observed_runtime_errors(token, accounts, group["id"], effective_platform) - return {**base, "ok": True, "operation": "errors", "mutation": False, "errors": errors} - detail = find_account(token, accounts, str(PAYLOAD["account"])) if action in ("get", "infrastructure") else None - if action == "get": - return {**base, "ok": True, "operation": "get", "mutation": False, "account": account_summary(detail, True)} - if action == "infrastructure": - return { - **base, - "ok": True, - "operation": "infrastructure", - "mutation": False, - "account": account_summary(detail, False), - "infrastructure": infrastructure_snapshot(token, detail), - } - include_plan_rules = PAYLOAD.get("full") is True - details = selected_account_details(token, accounts) - plans = [mutation_plan(item, action, include_plan_rules) for item in details] - changed = sum(1 for plan in plans if plan["change"] != "noop") - if PAYLOAD.get("confirm") is not True: - items = [] - for plan in plans: - item = {key: value for key, value in plan.items() if not key.startswith("_")} - item["write"] = {"attempted": False, "status": "dry-run", "error": None} - item["actual"] = item["before"] - item["reconciled"] = None - item["reconcileError"] = None - items.append(item) - summary = {"selected": len(items), "changed": changed, "writeSucceeded": 0, "writeFailed": 0, "reconciled": 0, "mismatched": 0} - return {**base, "ok": True, "operation": action, "mode": "dry-run", "mutation": False, "partialWrite": False, "summary": summary, "items": items, "credentialsPrinted": False} - writes = [write_mutation_plan(token, plan) for plan in plans] - items = [reconcile_mutation_plan(token, plan, write, include_plan_rules) for plan, write in zip(plans, writes)] - write_succeeded = sum(1 for item in items if item["write"]["status"] == "succeeded") - write_failed = sum(1 for item in items if item["write"]["status"] == "failed") - reconciled = sum(1 for item in items if item["reconciled"] is True) - mismatched = sum(1 for item in items if item["reconciled"] is False) - summary = {"selected": len(items), "changed": changed, "writeSucceeded": write_succeeded, "writeFailed": write_failed, "reconciled": reconciled, "mismatched": mismatched} - ok = write_failed == 0 and mismatched == 0 - return {**base, "ok": ok, "operation": action, "mode": "confirmed", "mutation": write_succeeded > 0, "partialWrite": write_succeeded > 0 and write_failed > 0, "summary": summary, "items": items, "credentialsPrinted": False} - -try: - result = runtime_result() -except Exception as exc: - result = {"ok": False, "operation": PAYLOAD.get("action"), "mutation": False, "error": str(exc), "valuesPrinted": False} - -print(json.dumps(result, ensure_ascii=False, indent=2)) -sys.exit(0 if result.get("ok") else 1) -PY -`; -}