diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index 97e1da13..f8e8f4ca 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -1870,6 +1870,7 @@ TRACE_PATH_TEMPLATE = ${JSON.stringify(observability.probes.traceQueryPathTempla FULL = ${options.full ? "True" : "False"} RAW = ${options.raw ? "True" : "False"} LIMIT = ${options.limit} +CANDIDATE_LIMIT = ${options.candidateLimit} DEADLINE = time.time() + 50 IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", @@ -2416,6 +2417,101 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error candidates.sort(key=lambda item: item.get("confidence", 0), reverse=True) return candidates +def candidate_score(trace_id, meta, trace_body, trace_rc, trace_err): + parsed = parse_json(trace_body) + meta_root_service = str(meta.get("rootServiceName") or "") if isinstance(meta, dict) else "" + meta_root_name = str(meta.get("rootTraceName") or "") if isinstance(meta, dict) else "" + if not isinstance(parsed, dict): + return { + "traceId": trace_id, + "score": -100, + "reasons": ["trace parse failed"], + "parseOk": False, + "queryOk": trace_rc == 0, + "meta": compact_meta(meta), + "spanCount": 0, + "services": [], + "servicePath": { + "hwlab-cloud-api": "unknown", + "agentrun-manager": "unknown", + "agentrun-runner": "unknown", + "complete": False, + }, + "rootTraceName": meta_root_name or None, + "rootServiceName": meta_root_service or None, + "stderrTail": (trace_err or "")[-1000:], + }, None + flat = flatten_trace(parsed) + spans = flat["spans"] + services = set(flat["services"]) + names = [str(item.get("name") or "") for item in spans] + lowered_names = [name.lower() for name in names] + identity = identity_from_spans(spans) + agentrun = agentrun_summary(spans) + error_spans = flat["errorSpans"] + score = 0 + reasons = [] + def add(points, reason): + nonlocal score + score += points + reasons.append("%+d %s" % (points, reason)) + if "agentrun-runner" in services: + add(120, "contains agentrun-runner") + if "agentrun-manager" in services: + add(90, "contains agentrun-manager") + if "hwlab-cloud-api" in services: + add(20, "contains hwlab-cloud-api") + span_points = min(len(spans), 80) + if span_points: + add(span_points, "span count %s" % len(spans)) + for key, points in (("runId", 50), ("commandId", 50), ("runnerJobId", 35), ("runnerId", 30), ("attemptId", 20), ("backendProfile", 15)): + if identity.get(key) not in (None, ""): + add(points, "identity %s" % key) + if any("codex_stdio" in name for name in lowered_names): + add(45, "codex stdio span") + if any("runner" in name for name in lowered_names): + add(30, "runner span") + if any("provider" in name for name in lowered_names): + add(25, "provider span") + if any("tool_call" in name or "command" in name for name in lowered_names): + add(20, "tool/command span") + if any("turn_completed" in name or "runner_terminal" in name or "command_result" in name for name in lowered_names): + add(35, "terminal/result span") + if error_spans: + add(40 + min(len(error_spans), 20), "error span count %s" % len(error_spans)) + if agentrun.get("terminalStatus") not in (None, ""): + add(25, "terminal status %s" % agentrun.get("terminalStatus")) + is_workbench_get_single_span = ( + len(spans) <= 1 + and services.issubset({"hwlab-cloud-api"}) + and meta_root_name.startswith("GET /v1/workbench/") + ) + if is_workbench_get_single_span: + add(-80, "single-span Workbench GET/SSE trace") + service_path = { + service: ("reached" if service in services else "missing") + for service in ("hwlab-cloud-api", "agentrun-manager", "agentrun-runner") + } + service_path["complete"] = all(service in services for service in ("hwlab-cloud-api", "agentrun-manager", "agentrun-runner")) + return { + "traceId": trace_id, + "score": score, + "reasons": reasons[:12], + "parseOk": True, + "queryOk": trace_rc == 0, + "meta": compact_meta(meta), + "spanCount": len(spans), + "services": sorted(services), + "servicePath": service_path, + "identity": {key: identity.get(key) for key in ("runId", "commandId", "runnerJobId", "runnerId", "backendProfile") if identity.get(key) not in (None, "")}, + "terminalStatus": agentrun.get("terminalStatus"), + "errorSpanCount": len(error_spans), + "rootTraceName": meta_root_name or None, + "rootServiceName": meta_root_service or None, + "spanNamePreview": names[:8], + "traceCommand": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % trace_id, + }, parsed + def resolve_trace(): if TRACE_ID is not None: return TRACE_ID, { @@ -2433,14 +2529,39 @@ def resolve_trace(): search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15) search_parsed = parse_json(search_body) metas = extract_traces(search_parsed) + candidate_summaries = [] selected = None selected_trace_id = None + selected_score = None + seen = set() + scanned = 0 for meta in metas: trace_id = trace_id_from_meta(meta) - if trace_id: + if not trace_id or trace_id in seen: + continue + seen.add(trace_id) + if scanned >= CANDIDATE_LIMIT or time.time() > DEADLINE: + break + scanned += 1 + trace_path = TRACE_PATH_TEMPLATE.replace("{{traceId}}", trace_id) + trace_rc, trace_body, trace_err = run_kubectl(TRACE_PROXY_PREFIX + trace_path, timeout=8) + summary, _parsed = candidate_score(trace_id, meta, trace_body, trace_rc, trace_err) + candidate_summaries.append(summary) + score = summary.get("score") + if isinstance(score, (int, float)) and (selected_score is None or score > selected_score): selected = meta selected_trace_id = trace_id - break + selected_score = score + if selected_trace_id is None: + for meta in metas: + trace_id = trace_id_from_meta(meta) + if trace_id: + selected = meta + selected_trace_id = trace_id + selected_score = None + break + candidate_summaries.sort(key=lambda item: item.get("score", -999999), reverse=True) + selected_summary = next((item for item in candidate_summaries if item.get("traceId") == selected_trace_id), None) mapping = { "mode": "business-trace-id", "businessTraceId": BUSINESS_TRACE_ID, @@ -2450,7 +2571,12 @@ def resolve_trace(): "searchOk": search_rc == 0, "searchParseOk": isinstance(search_parsed, dict), "candidateTraceCount": len(metas), + "scannedCandidateCount": scanned, + "candidateSelectionMode": "scored-service-and-span-content", + "selectedScore": selected_score, + "selectedReasons": selected_summary.get("reasons", []) if isinstance(selected_summary, dict) else [], "selectedMeta": compact_meta(selected), + "candidatePreview": candidate_summaries[:8], "searchStderrTail": (search_err or "")[-2000:], } if RAW: