mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: score OTel code agent trace candidates (#570)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-21 13:04:35 +08:00
committed by GitHub
parent d25580c57c
commit e3dd4b6409
1 changed files with 128 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-observability.ts
+128 -2
View File
@@ -1870,6 +1870,7 @@ TRACE_PATH_TEMPLATE = ${JSON.stringify(observability.probes.traceQueryPathTempla
FULL = ${options.full ? "True" : "False"}
RAW = ${options.raw ? "True" : "False"}
LIMIT = ${options.limit}
CANDIDATE_LIMIT = ${options.candidateLimit}
DEADLINE = time.time() + 50
IMPORTANT_ATTRS = [
"traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
@@ -2416,6 +2417,101 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error
candidates.sort(key=lambda item: item.get("confidence", 0), reverse=True)
return candidates
def candidate_score(trace_id, meta, trace_body, trace_rc, trace_err):
parsed = parse_json(trace_body)
meta_root_service = str(meta.get("rootServiceName") or "") if isinstance(meta, dict) else ""
meta_root_name = str(meta.get("rootTraceName") or "") if isinstance(meta, dict) else ""
if not isinstance(parsed, dict):
return {
"traceId": trace_id,
"score": -100,
"reasons": ["trace parse failed"],
"parseOk": False,
"queryOk": trace_rc == 0,
"meta": compact_meta(meta),
"spanCount": 0,
"services": [],
"servicePath": {
"hwlab-cloud-api": "unknown",
"agentrun-manager": "unknown",
"agentrun-runner": "unknown",
"complete": False,
},
"rootTraceName": meta_root_name or None,
"rootServiceName": meta_root_service or None,
"stderrTail": (trace_err or "")[-1000:],
}, None
flat = flatten_trace(parsed)
spans = flat["spans"]
services = set(flat["services"])
names = [str(item.get("name") or "") for item in spans]
lowered_names = [name.lower() for name in names]
identity = identity_from_spans(spans)
agentrun = agentrun_summary(spans)
error_spans = flat["errorSpans"]
score = 0
reasons = []
def add(points, reason):
nonlocal score
score += points
reasons.append("%+d %s" % (points, reason))
if "agentrun-runner" in services:
add(120, "contains agentrun-runner")
if "agentrun-manager" in services:
add(90, "contains agentrun-manager")
if "hwlab-cloud-api" in services:
add(20, "contains hwlab-cloud-api")
span_points = min(len(spans), 80)
if span_points:
add(span_points, "span count %s" % len(spans))
for key, points in (("runId", 50), ("commandId", 50), ("runnerJobId", 35), ("runnerId", 30), ("attemptId", 20), ("backendProfile", 15)):
if identity.get(key) not in (None, ""):
add(points, "identity %s" % key)
if any("codex_stdio" in name for name in lowered_names):
add(45, "codex stdio span")
if any("runner" in name for name in lowered_names):
add(30, "runner span")
if any("provider" in name for name in lowered_names):
add(25, "provider span")
if any("tool_call" in name or "command" in name for name in lowered_names):
add(20, "tool/command span")
if any("turn_completed" in name or "runner_terminal" in name or "command_result" in name for name in lowered_names):
add(35, "terminal/result span")
if error_spans:
add(40 + min(len(error_spans), 20), "error span count %s" % len(error_spans))
if agentrun.get("terminalStatus") not in (None, ""):
add(25, "terminal status %s" % agentrun.get("terminalStatus"))
is_workbench_get_single_span = (
len(spans) <= 1
and services.issubset({"hwlab-cloud-api"})
and meta_root_name.startswith("GET /v1/workbench/")
)
if is_workbench_get_single_span:
add(-80, "single-span Workbench GET/SSE trace")
service_path = {
service: ("reached" if service in services else "missing")
for service in ("hwlab-cloud-api", "agentrun-manager", "agentrun-runner")
}
service_path["complete"] = all(service in services for service in ("hwlab-cloud-api", "agentrun-manager", "agentrun-runner"))
return {
"traceId": trace_id,
"score": score,
"reasons": reasons[:12],
"parseOk": True,
"queryOk": trace_rc == 0,
"meta": compact_meta(meta),
"spanCount": len(spans),
"services": sorted(services),
"servicePath": service_path,
"identity": {key: identity.get(key) for key in ("runId", "commandId", "runnerJobId", "runnerId", "backendProfile") if identity.get(key) not in (None, "")},
"terminalStatus": agentrun.get("terminalStatus"),
"errorSpanCount": len(error_spans),
"rootTraceName": meta_root_name or None,
"rootServiceName": meta_root_service or None,
"spanNamePreview": names[:8],
"traceCommand": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % trace_id,
}, parsed
def resolve_trace():
if TRACE_ID is not None:
return TRACE_ID, {
@@ -2433,14 +2529,39 @@ def resolve_trace():
search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15)
search_parsed = parse_json(search_body)
metas = extract_traces(search_parsed)
candidate_summaries = []
selected = None
selected_trace_id = None
selected_score = None
seen = set()
scanned = 0
for meta in metas:
trace_id = trace_id_from_meta(meta)
if trace_id:
if not trace_id or trace_id in seen:
continue
seen.add(trace_id)
if scanned >= CANDIDATE_LIMIT or time.time() > DEADLINE:
break
scanned += 1
trace_path = TRACE_PATH_TEMPLATE.replace("{{traceId}}", trace_id)
trace_rc, trace_body, trace_err = run_kubectl(TRACE_PROXY_PREFIX + trace_path, timeout=8)
summary, _parsed = candidate_score(trace_id, meta, trace_body, trace_rc, trace_err)
candidate_summaries.append(summary)
score = summary.get("score")
if isinstance(score, (int, float)) and (selected_score is None or score > selected_score):
selected = meta
selected_trace_id = trace_id
break
selected_score = score
if selected_trace_id is None:
for meta in metas:
trace_id = trace_id_from_meta(meta)
if trace_id:
selected = meta
selected_trace_id = trace_id
selected_score = None
break
candidate_summaries.sort(key=lambda item: item.get("score", -999999), reverse=True)
selected_summary = next((item for item in candidate_summaries if item.get("traceId") == selected_trace_id), None)
mapping = {
"mode": "business-trace-id",
"businessTraceId": BUSINESS_TRACE_ID,
@@ -2450,7 +2571,12 @@ def resolve_trace():
"searchOk": search_rc == 0,
"searchParseOk": isinstance(search_parsed, dict),
"candidateTraceCount": len(metas),
"scannedCandidateCount": scanned,
"candidateSelectionMode": "scored-service-and-span-content",
"selectedScore": selected_score,
"selectedReasons": selected_summary.get("reasons", []) if isinstance(selected_summary, dict) else [],
"selectedMeta": compact_meta(selected),
"candidatePreview": candidate_summaries[:8],
"searchStderrTail": (search_err or "")[-2000:],
}
if RAW: