diff --git a/scripts/src/platform-infra-sub2api-codex/faults.ts b/scripts/src/platform-infra-sub2api-codex/faults.ts index 8f89e410..61e276bd 100644 --- a/scripts/src/platform-infra-sub2api-codex/faults.ts +++ b/scripts/src/platform-infra-sub2api-codex/faults.ts @@ -397,6 +397,19 @@ def items_of(data): def total_of(data): return int(data.get("total", len(items_of(data)))) if isinstance(data, dict) else len(items_of(data)) +def all_items(token, path, params, label): + page = 1 + page_size = 500 + rows = [] + while True: + data = get(token, path, {**params, "page": page, "page_size": page_size}, label) + batch = items_of(data) + rows.extend(batch) + total = total_of(data) + if not batch or len(rows) >= total: + return rows + page += 1 + def percent(value): if not isinstance(value, (int, float)): return None @@ -417,8 +430,6 @@ def filters_for(group_id=None): result["stream"] = "true" if filters["stream"] == "stream" else "false" if filters.get("endpoint"): result["inbound_endpoint"] = filters["endpoint"] - if filters.get("requestId"): - result["request_id"] = filters["requestId"] return result def selected_groups(groups): @@ -510,14 +521,30 @@ def execute(): for group, metrics in group_data: group_id = group.get("id") endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors" - data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") - total += total_of(data) + request_id = PAYLOAD["filters"].get("requestId") + if request_id: + rows = [ + item for item in all_items(token, endpoint, {**filters_for(group_id), "view": "all", "include_detail": 1}, level + " request lookup") + if item.get("request_id") == request_id or item.get("client_request_id") == request_id + ] + total += len(rows) + rows = rows[offset:offset + PAYLOAD["pageSize"]] + else: + data = get(token, endpoint, {**filters_for(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details") + total += total_of(data) + rows = items_of(data) availability = availability_by_account(token, group_id) if level == "P2" else {} - for item in items_of(data): + customer_request_ids = set() + if level == "P2": + customer_request_ids = { + item.get("request_id") or item.get("client_request_id") + for item in all_items(token, "/api/v1/admin/ops/request-errors", {**filters_for(group_id), "view": "all"}, "customer error correlation") + if item.get("request_id") or item.get("client_request_id") + } + for item in rows: absorbed = None if level == "P2" and item.get("request_id"): - correlated = get(token, "/api/v1/admin/ops/request-errors", {"time_range": "24h", "request_id": item.get("request_id"), "page": 1, "page_size": 1, "view": "all"}, "correlated request error") - absorbed = total_of(correlated) == 0 + absorbed = item.get("request_id") not in customer_request_ids details.append(detail_row(item, metrics, availability, level, absorbed)) detail_columns = ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "CUSTOMER_ERR%", "ABSORBED", "REQUEST_ID"] if level == "P0" else ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "ROOT", "TEMP_UNSCHED", "ABSORBED", "REQUEST_ID"] elif level == "P1":