diff --git a/scripts/src/platform-infra-sub2api-codex/faults.ts b/scripts/src/platform-infra-sub2api-codex/faults.ts index 61e276bd..57e72890 100644 --- a/scripts/src/platform-infra-sub2api-codex/faults.ts +++ b/scripts/src/platform-infra-sub2api-codex/faults.ts @@ -218,11 +218,11 @@ function renderFaults(response: Record): RenderedCliResult { const summary = arrayOfRecords(faults.summary); if (summary.length > 0) { lines.push("GROUPS"); - lines.push(table(["GROUP", "P0", "CUSTOMER_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERR%", "ABSORB%"], summary.map((row) => [ + lines.push(table(["GROUP", "P0", "CUSTOMER_ERRS", "GROUP_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERRS", "GROUP_UP_ERR%", "ABSORB%"], summary.map((row) => [ `${text(row.groupName)} (${text(row.groupId)})`, - text(row.p0), percent(row.customerErrorRatePercent), + text(row.p0), text(row.customerErrorCount), percent(row.customerErrorRatePercent), text(row.p1), milliseconds(row.ttftP99Ms), - text(row.p2), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent), + text(row.p2), text(row.upstreamErrorCount), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent), ]))); } const details = arrayOfRecords(faults.details); @@ -418,19 +418,43 @@ def percent(value): def selector_id(value): return int(value) if isinstance(value, str) and value.isdigit() else None -def filters_for(group_id=None): - filters = PAYLOAD["filters"] +def scope_filters(group_id=None): result = {"time_range": "24h", "platform": "openai", "group_id": group_id} + return result + +def matches_detail_filters(item): + filters = PAYLOAD["filters"] account = filters.get("account") if account: - result["account_id" if selector_id(account) is not None else "account_name"] = selector_id(account) if selector_id(account) is not None else account - if filters.get("model"): - result["model"] = filters["model"] + account_match = str(item.get("account_id") or "") == str(account) or str(item.get("account_name") or "") == str(account) + if not account_match: + return False + if filters.get("model") and str(item.get("requested_model") or item.get("model") or "") != str(filters["model"]): + return False if filters.get("stream"): - result["stream"] = "true" if filters["stream"] == "stream" else "false" - if filters.get("endpoint"): - result["inbound_endpoint"] = filters["endpoint"] - return result + expected_stream = filters["stream"] == "stream" + if item.get("stream") is not expected_stream: + return False + if filters.get("endpoint") and str(item.get("inbound_endpoint") or item.get("request_path") or item.get("path") or "") != str(filters["endpoint"]): + return False + if filters.get("requestId"): + request_id = filters["requestId"] + if item.get("request_id") != request_id and item.get("client_request_id") != request_id: + return False + return True + +def has_detail_filters(): + filters = PAYLOAD["filters"] + return any(filters.get(key) for key in ("account", "model", "stream", "endpoint", "requestId")) + +DETAIL_ROWS = {} + +def error_rows(token, path, group_id, apply_filters): + cache_key = (path, group_id, apply_filters) + if cache_key not in DETAIL_ROWS: + rows = all_items(token, path, {**scope_filters(group_id), "view": "all", "include_detail": 1}, "fault detail scan") + DETAIL_ROWS[cache_key] = [item for item in rows if matches_detail_filters(item)] if apply_filters else rows + return DETAIL_ROWS[cache_key] def selected_groups(groups): selector = PAYLOAD["filters"].get("group") @@ -486,10 +510,14 @@ def execute(): for group in groups: group_id = group.get("id") native = get(token, "/api/v1/admin/ops/dashboard/overview", {"time_range": "24h", "platform": "openai", "group_id": group_id}, "dashboard overview") - request_errors = get(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors") - upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**filters_for(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors") - customer_total = total_of(request_errors) - upstream_total = total_of(upstream_errors) + if has_detail_filters(): + customer_total = len(error_rows(token, "/api/v1/admin/ops/request-errors", group_id, True)) + upstream_total = len(error_rows(token, "/api/v1/admin/ops/upstream-errors", group_id, True)) + else: + request_errors = get(token, "/api/v1/admin/ops/request-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors") + upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors") + customer_total = total_of(request_errors) + upstream_total = total_of(upstream_errors) absorbed = max(0, upstream_total - customer_total) absorbed_percent = round(absorbed * 100 / upstream_total, 2) if upstream_total else None customer_rate = percent(native.get("error_rate") if isinstance(native, dict) else None) @@ -521,16 +549,12 @@ def execute(): for group, metrics in group_data: group_id = group.get("id") endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors" - request_id = PAYLOAD["filters"].get("requestId") - if request_id: - rows = [ - item for item in all_items(token, endpoint, {**filters_for(group_id), "view": "all", "include_detail": 1}, level + " request lookup") - if item.get("request_id") == request_id or item.get("client_request_id") == request_id - ] + if has_detail_filters(): + rows = error_rows(token, endpoint, group_id, True) total += len(rows) rows = rows[offset:offset + PAYLOAD["pageSize"]] else: - data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") + data = get(token, endpoint, {**scope_filters(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") total += total_of(data) rows = items_of(data) availability = availability_by_account(token, group_id) if level == "P2" else {} @@ -538,7 +562,7 @@ def execute(): if level == "P2": customer_request_ids = { item.get("request_id") or item.get("client_request_id") - for item in all_items(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "view": "all"}, "customer error correlation") + for item in error_rows(token, "/api/v1/admin/ops/request-errors", group_id, False) if item.get("request_id") or item.get("client_request_id") } for item in rows: