feat: improve sub2api recent error analysis
This commit is contained in:
@@ -3006,6 +3006,16 @@ def trace_untried_schedulable_accounts(failovers, final_event, account_snapshot)
|
||||
})
|
||||
return result
|
||||
|
||||
def ops_start_time_for_since(since):
|
||||
if not isinstance(since, str):
|
||||
return None
|
||||
match = re.fullmatch(r"\s*(\d+)\s*([smhd])\s*", since.lower())
|
||||
if match is None:
|
||||
return None
|
||||
value = int(match.group(1))
|
||||
multiplier = {"s": 1, "m": 60, "h": 3600, "d": 86400}[match.group(2)]
|
||||
return (datetime.now(timezone.utc) - timedelta(seconds=value * multiplier)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
def run_trace():
|
||||
payload = json.loads(base64.b64decode(PAYLOAD_B64).decode("utf-8")) if PAYLOAD_B64 else {}
|
||||
request_id = payload.get("requestId")
|
||||
@@ -3032,6 +3042,35 @@ def run_trace():
|
||||
"returned": len(request_items),
|
||||
"total": request_data.get("total", len(request_items)) if isinstance(request_data, dict) else len(request_items),
|
||||
}
|
||||
if native_request is None:
|
||||
ops_start_time = ops_start_time_for_since(since)
|
||||
error_query = f"/api/v1/admin/ops/request-errors?q={request_query}&view=all&page=1&page_size=20"
|
||||
if isinstance(ops_start_time, str):
|
||||
error_query += "&start_time=" + quote(ops_start_time, safe="")
|
||||
error_data = ensure_success(
|
||||
curl_api("GET", error_query, bearer=token),
|
||||
"get ops request error fallback",
|
||||
)
|
||||
error_items = extract_items(error_data)
|
||||
native_request = next((item for item in error_items if isinstance(item, dict) and item.get("request_id") == request_id), None)
|
||||
native_request_status.update({
|
||||
"fallback": "sub2api-native-admin-ops-request-errors",
|
||||
"fallbackStartTime": ops_start_time,
|
||||
"fallbackReturned": len(error_items),
|
||||
"fallbackTotal": error_data.get("total", len(error_items)) if isinstance(error_data, dict) else len(error_items),
|
||||
})
|
||||
if isinstance(native_request, dict):
|
||||
native_request = dict(native_request)
|
||||
native_request["kind"] = "error"
|
||||
error_id = native_request.get("id")
|
||||
if isinstance(error_id, int):
|
||||
try:
|
||||
detail = ensure_success(curl_api("GET", f"/api/v1/admin/ops/request-errors/{error_id}", bearer=token), "get ops request error detail")
|
||||
if isinstance(detail, dict):
|
||||
native_request.update(detail)
|
||||
except Exception:
|
||||
pass
|
||||
native_request_status.update({"matched": True, "source": "sub2api-native-admin-ops-request-errors"})
|
||||
except Exception as exc:
|
||||
native_request_status = {"status": "unavailable", "reason": text(str(exc), 500)}
|
||||
|
||||
@@ -3119,7 +3158,7 @@ def run_trace():
|
||||
"type": "request-start",
|
||||
"at": native_request.get("created_at"),
|
||||
"requestId": native_request.get("request_id"),
|
||||
"path": None,
|
||||
"path": native_request.get("request_path") or native_request.get("inbound_endpoint"),
|
||||
"model": native_request.get("model"),
|
||||
"accountId": native_request.get("account_id"),
|
||||
"accountName": account_names_by_id.get(native_request.get("account_id")),
|
||||
@@ -3137,7 +3176,7 @@ def run_trace():
|
||||
"type": "final",
|
||||
"at": native_request.get("created_at"),
|
||||
"requestId": native_request.get("request_id"),
|
||||
"path": None,
|
||||
"path": native_request.get("request_path") or native_request.get("inbound_endpoint"),
|
||||
"model": native_request.get("model"),
|
||||
"accountId": native_request.get("account_id"),
|
||||
"accountName": account_names_by_id.get(native_request.get("account_id")),
|
||||
@@ -3182,7 +3221,7 @@ def run_trace():
|
||||
final_elapsed = round((final_epoch - reference_epoch) * 1000) if final_epoch is not None and reference_epoch is not None else None
|
||||
failover_to_final = final_elapsed - first_failover_elapsed if isinstance(final_elapsed, int) and isinstance(first_failover_elapsed, int) else None
|
||||
reason = trace_reason(events, final_event)
|
||||
if not matched:
|
||||
if not matched and not isinstance(native_request, dict):
|
||||
outcome = "not-found"
|
||||
elif isinstance(final_event, dict) and isinstance(final_event.get("statusCode"), int) and final_event.get("statusCode") < 400:
|
||||
outcome = "degraded" if forward_failures else ("succeeded-with-failover" if failovers else "succeeded")
|
||||
@@ -3191,7 +3230,7 @@ def run_trace():
|
||||
else:
|
||||
outcome = "incomplete"
|
||||
return {
|
||||
"ok": proc.returncode == 0 and len(matched) > 0,
|
||||
"ok": proc.returncode == 0 and (len(matched) > 0 or isinstance(native_request, dict)),
|
||||
"mode": "trace",
|
||||
"targetId": TARGET_ID,
|
||||
"runtimeMode": RUNTIME_MODE,
|
||||
|
||||
Reference in New Issue
Block a user