fix cli output dumping and otel web-probe diagnostics
This commit is contained in:
Codex
@@ -4,6 +4,7 @@ import { Buffer } from "node:buffer";
|
||||
import { readFileSync } from "node:fs";
|
||||
import type { UniDeskConfig } from "./config";
|
||||
import { rootPath } from "./config";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import {
|
||||
compactCapture,
|
||||
createYamlFieldReader,
|
||||
@@ -269,9 +270,9 @@ function parseSearchOptions(args: string[]): SearchOptions {
|
||||
let lookbackMinutes = 360;
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--grep") {
|
||||
if (arg === "--grep" || arg === "--path") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error("--grep requires a value");
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
|
||||
grep = value;
|
||||
index += 1;
|
||||
} else if (arg === "--query" || arg === "--q") {
|
||||
@@ -620,14 +621,14 @@ async function validate(config: UniDeskConfig, options: CommonOptions): Promise<
|
||||
};
|
||||
}
|
||||
|
||||
async function trace(config: UniDeskConfig, options: TraceOptions): Promise<Record<string, unknown>> {
|
||||
async function trace(config: UniDeskConfig, options: TraceOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
if (options.traceId === null) throw new Error("observability trace requires --trace-id <traceId>");
|
||||
const observability = readObservabilityConfig();
|
||||
const target = resolveTarget(observability, options.targetId);
|
||||
const tracePath = observability.probes.traceQueryPathTemplate.replaceAll("{{traceId}}", encodeURIComponent(options.traceId));
|
||||
const result = await capture(config, target.route, ["sh"], traceScript(observability, target, tracePath, options));
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
return {
|
||||
const response = {
|
||||
ok: result.exitCode === 0 && parsed?.ok === true,
|
||||
action: "platform-infra-observability-trace",
|
||||
mutation: false,
|
||||
@@ -640,9 +641,11 @@ async function trace(config: UniDeskConfig, options: TraceOptions): Promise<Reco
|
||||
},
|
||||
result: redactSensitiveUnknown(parsed),
|
||||
};
|
||||
if (options.raw || options.full) return response;
|
||||
return renderTraceResult(response);
|
||||
}
|
||||
|
||||
async function search(config: UniDeskConfig, options: SearchOptions): Promise<Record<string, unknown>> {
|
||||
async function search(config: UniDeskConfig, options: SearchOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const observability = readObservabilityConfig();
|
||||
const target = resolveTarget(observability, options.targetId);
|
||||
const endSeconds = Math.floor(Date.now() / 1000);
|
||||
@@ -656,7 +659,7 @@ async function search(config: UniDeskConfig, options: SearchOptions): Promise<Re
|
||||
const searchPath = `/api/search?${params.toString()}`;
|
||||
const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, options));
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
return {
|
||||
const response = {
|
||||
ok: result.exitCode === 0 && parsed?.ok === true,
|
||||
action: "platform-infra-observability-search",
|
||||
mutation: false,
|
||||
@@ -673,9 +676,11 @@ async function search(config: UniDeskConfig, options: SearchOptions): Promise<Re
|
||||
},
|
||||
result: redactSensitiveUnknown(parsed) ?? compactCapture(result, { full: true }),
|
||||
};
|
||||
if (options.raw || options.full) return response;
|
||||
return renderSearchResult(response);
|
||||
}
|
||||
|
||||
async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise<Record<string, unknown>> {
|
||||
async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const observability = readObservabilityConfig();
|
||||
const target = resolveTarget(observability, options.targetId);
|
||||
const endSeconds = Math.floor(Date.now() / 1000);
|
||||
@@ -692,7 +697,7 @@ async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAge
|
||||
}
|
||||
const result = await capture(config, target.route, ["sh"], diagnoseCodeAgentScript(observability, target, searchPath, options));
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
return {
|
||||
const response = {
|
||||
ok: result.exitCode === 0 && parsed?.ok === true,
|
||||
action: "platform-infra-observability-diagnose-code-agent",
|
||||
mutation: false,
|
||||
@@ -709,6 +714,220 @@ async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAge
|
||||
},
|
||||
result: redactSensitiveUnknown(parsed) ?? compactCapture(result, { full: true }),
|
||||
};
|
||||
if (options.raw || options.full) return response;
|
||||
return renderDiagnoseCodeAgentResult(response);
|
||||
}
|
||||
|
||||
function renderDiagnoseCodeAgentResult(response: Record<string, unknown>): RenderedCliResult {
|
||||
const target = recordValue(response.target);
|
||||
const query = recordValue(response.query);
|
||||
const result = recordValue(response.result);
|
||||
const mapping = recordValue(result.mapping);
|
||||
const summary = recordValue(result.summary);
|
||||
const identity = recordValue(result.identity);
|
||||
const agentrun = recordValue(result.agentrun);
|
||||
const http = recordValue(result.http);
|
||||
const servicePath = recordValue(result.servicePath);
|
||||
const selection = recordValue(mapping.selection);
|
||||
const candidates = arrayValue(mapping.candidateScores);
|
||||
const rootCandidates = arrayValue(summary.rootCauseCandidates);
|
||||
const problemCounts = arrayValue(http.problemCounts);
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push("COMMAND TARGET OK BUSINESS_TRACE OTEL_TRACE");
|
||||
lines.push(`${pad("platform-infra observability diagnose", 44)} ${pad(textValue(target.id) || "-", 7)} ${pad(response.ok === true ? "true" : "false", 6)} ${pad(textValue(query.businessTraceId) || "-", 27)} ${textValue(mapping.otelTraceId) || "-"}`);
|
||||
lines.push("");
|
||||
lines.push("TRACE_SELECTION");
|
||||
lines.push(`mode: ${textValue(selection.mode) || textValue(mapping.mode) || "-"}`);
|
||||
lines.push(`candidateTraceCount: ${textValue(mapping.candidateTraceCount) || "-"}`);
|
||||
lines.push(`selectedScore: ${textValue(selection.score) || "-"}`);
|
||||
lines.push(`selectedReason: ${arrayValue(selection.reasons).map((item) => textValue(item)).filter(Boolean).join(", ") || "-"}`);
|
||||
if (textValue(mapping.selectionWarning)) lines.push(`warning: ${textValue(mapping.selectionWarning)}`);
|
||||
lines.push("");
|
||||
lines.push("SERVICE_PATH");
|
||||
for (const service of ["hwlab-cloud-api", "agentrun-manager", "agentrun-runner"]) {
|
||||
lines.push(`${pad(service, 22)} ${textValue(servicePath[service]) || "missing"}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("AGENTRUN");
|
||||
lines.push(`terminalStatus: ${textValue(agentrun.terminalStatus) || "-"}`);
|
||||
lines.push(`terminalEventType: ${textValue(agentrun.terminalEventType) || "-"}`);
|
||||
lines.push(`runnerProviderClassification: ${textValue(agentrun.runnerProviderClassification) || "-"}`);
|
||||
lines.push("");
|
||||
lines.push("IDENTITY");
|
||||
for (const key of ["runId", "commandId", "runnerJobId", "runnerId", "sessionId", "turnId", "backendProfile", "sourceCommit"]) {
|
||||
const value = textValue(identity[key]);
|
||||
if (value) lines.push(`${pad(key, 16)} ${value}`);
|
||||
}
|
||||
if (lines[lines.length - 1] === "IDENTITY") lines.push("-");
|
||||
lines.push("");
|
||||
lines.push("ROOT_CAUSE");
|
||||
lines.push(textValue(summary.rootCause) || "-");
|
||||
for (const item of rootCandidates.slice(0, 5)) {
|
||||
const row = recordValue(item);
|
||||
lines.push(`- ${textValue(row.code) || textValue(row.label) || "candidate"} confidence=${textValue(row.confidence) || "-"} ${textValue(row.summary) || ""}`.trim());
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("HTTP_PROBLEMS");
|
||||
if (problemCounts.length === 0) {
|
||||
lines.push("-");
|
||||
} else {
|
||||
lines.push("METHOD ROUTE STATUS COUNT");
|
||||
for (const item of problemCounts.slice(0, 8)) {
|
||||
const row = recordValue(item);
|
||||
lines.push(`${pad(textValue(row.method) || "-", 7)} ${pad(truncate(textValue(row.route) || "-", 29), 29)} ${pad(textValue(row.status) || "-", 7)} ${textValue(row.count) || "-"}`);
|
||||
}
|
||||
}
|
||||
if (candidates.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("CANDIDATE_TRACE_SCORES");
|
||||
lines.push("SCORE SPANS SERVICES TRACE");
|
||||
for (const item of candidates.slice(0, 8)) {
|
||||
const row = recordValue(item);
|
||||
lines.push(`${pad(textValue(row.score) || "0", 6)} ${pad(textValue(row.spanCount) || "-", 6)} ${pad(truncate(arrayValue(row.services).map((value) => textValue(value)).filter(Boolean).join(","), 32), 32)} ${textValue(row.traceId) || "-"}`);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("Next:");
|
||||
if (textValue(mapping.otelTraceId)) {
|
||||
lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${textValue(target.id) || textValue(query.target) || "D601"} --trace-id ${textValue(mapping.otelTraceId)} --limit 40`);
|
||||
lines.push(` bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${textValue(target.id) || "D601"} --business-trace-id ${textValue(query.businessTraceId) || "<trc_...>"} --full`);
|
||||
} else {
|
||||
lines.push(` bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${textValue(target.id) || "D601"} --business-trace-id ${textValue(query.businessTraceId) || "<trc_...>"} --lookback-minutes 10080 --candidate-limit 100`);
|
||||
}
|
||||
return {
|
||||
ok: response.ok === true,
|
||||
command: "platform-infra observability diagnose-code-agent",
|
||||
contentType: "text/plain",
|
||||
renderedText: `${lines.join("\n")}\n`,
|
||||
};
|
||||
}
|
||||
|
||||
function renderSearchResult(response: Record<string, unknown>): RenderedCliResult {
|
||||
const target = recordValue(response.target);
|
||||
const query = recordValue(response.query);
|
||||
const result = recordValue(response.result);
|
||||
const traces = arrayValue(result.traces);
|
||||
const lines: string[] = [];
|
||||
lines.push("COMMAND TARGET OK GREP_OR_QUERY MATCHED SCANNED CANDIDATES");
|
||||
lines.push(`${pad("platform-infra observability search", 38)} ${pad(textValue(target.id) || "-", 7)} ${pad(response.ok === true ? "true" : "false", 6)} ${pad(truncate(textValue(query.grep) || textValue(query.tempoQuery) || "-", 35), 35)} ${pad(textValue(result.matchedTraceCount) || "-", 8)} ${pad(textValue(result.scannedTraceCount) || "-", 8)} ${textValue(result.candidateTraceCount) || "-"}`);
|
||||
if (textValue(result.scanStopped)) lines.push(`warning: scanStopped=${textValue(result.scanStopped)}`);
|
||||
const truncated = recordValue(result.truncated);
|
||||
if (truncated.candidateTraces === true || truncated.matchedTraces === true) {
|
||||
lines.push(`warning: truncated candidateTraces=${textValue(truncated.candidateTraces)} matchedTraces=${textValue(truncated.matchedTraces)}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("TRACES");
|
||||
if (traces.length === 0) {
|
||||
lines.push("-");
|
||||
} else {
|
||||
lines.push("TRACE SPANS ERRORS MATCH SERVICES ROOT");
|
||||
for (const item of traces.slice(0, 20)) {
|
||||
const row = recordValue(item);
|
||||
const meta = recordValue(row.meta);
|
||||
const services = arrayValue(row.services).map((value) => textValue(value)).filter(Boolean).join(",");
|
||||
lines.push(`${pad(textValue(row.traceId) || "-", 34)} ${pad(textValue(row.spanCount) || "-", 6)} ${pad(textValue(row.errorSpanCount) || "-", 7)} ${pad(textValue(row.matchedSpanCount) || "-", 6)} ${pad(truncate(services || textValue(meta.rootServiceName) || "-", 32), 32)} ${truncate(textValue(meta.rootTraceName) || "-", 48)}`);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("Next:");
|
||||
lines.push(` bun scripts/cli.ts platform-infra observability search --target ${textValue(target.id) || "D601"} --grep <text> --lookback-minutes ${textValue(query.lookbackMinutes) || "360"} --candidate-limit ${textValue(query.candidateLimit) || "80"} --limit ${textValue(query.limit) || "20"} --full`);
|
||||
lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${textValue(target.id) || "D601"} --trace-id <traceId> --grep <text> --limit 80`);
|
||||
return {
|
||||
ok: response.ok === true,
|
||||
command: "platform-infra observability search",
|
||||
contentType: "text/plain",
|
||||
renderedText: `${lines.join("\n")}\n`,
|
||||
};
|
||||
}
|
||||
|
||||
function renderTraceResult(response: Record<string, unknown>): RenderedCliResult {
|
||||
const target = recordValue(response.target);
|
||||
const traceId = textValue(response.traceId);
|
||||
const result = recordValue(response.result);
|
||||
const services = arrayValue(result.services).map((value) => textValue(value)).filter(Boolean);
|
||||
const businessTraceIds = arrayValue(result.businessTraceIds).map((value) => textValue(value)).filter(Boolean);
|
||||
const spanNameCounts = arrayValue(result.spanNameCounts);
|
||||
const errorSpans = arrayValue(result.errorSpans);
|
||||
const matchedSpans = arrayValue(result.spans).length > 0 ? arrayValue(result.spans) : arrayValue(result.matchedSpans);
|
||||
const lines: string[] = [];
|
||||
lines.push("COMMAND TARGET OK TRACE SPANS ERRORS MATCH");
|
||||
lines.push(`${pad("platform-infra observability trace", 37)} ${pad(textValue(target.id) || "-", 7)} ${pad(response.ok === true ? "true" : "false", 6)} ${pad(traceId || "-", 34)} ${pad(textValue(result.spanCount) || "-", 6)} ${pad(textValue(result.errorSpanCount) || "-", 7)} ${textValue(result.matchedSpanCount) || "-"}`);
|
||||
lines.push("");
|
||||
lines.push(`services: ${services.join(", ") || "-"}`);
|
||||
lines.push(`businessTraceIds: ${businessTraceIds.join(", ") || "-"}`);
|
||||
if (textValue(result.grep)) lines.push(`grep: ${textValue(result.grep)}`);
|
||||
const truncated = recordValue(result.truncated);
|
||||
if (truncated.errorSpans === true || truncated.spans === true) {
|
||||
lines.push(`warning: truncated errorSpans=${textValue(truncated.errorSpans)} spans=${textValue(truncated.spans)}`);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("SPAN_NAME_COUNTS");
|
||||
if (spanNameCounts.length === 0) {
|
||||
lines.push("-");
|
||||
} else {
|
||||
lines.push("COUNT NAME");
|
||||
for (const item of spanNameCounts.slice(0, 12)) {
|
||||
const row = recordValue(item);
|
||||
lines.push(`${pad(textValue(row.count) || "-", 6)} ${truncate(textValue(row.name) || "-", 80)}`);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("ERROR_SPANS");
|
||||
if (errorSpans.length === 0) {
|
||||
lines.push("-");
|
||||
} else {
|
||||
lines.push("SERVICE DURATION_MS NAME");
|
||||
for (const item of errorSpans.slice(0, 8)) {
|
||||
const row = recordValue(item);
|
||||
lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${truncate(textValue(row.name) || "-", 80)}`);
|
||||
}
|
||||
}
|
||||
if (matchedSpans.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("MATCHED_SPANS");
|
||||
lines.push("SERVICE DURATION_MS NAME");
|
||||
for (const item of matchedSpans.slice(0, 8)) {
|
||||
const row = recordValue(item);
|
||||
lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${truncate(textValue(row.name) || "-", 80)}`);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("Next:");
|
||||
lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${textValue(target.id) || "D601"} --trace-id ${traceId || "<otelTraceId>"} --grep <text> --limit 80`);
|
||||
lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${textValue(target.id) || "D601"} --trace-id ${traceId || "<otelTraceId>"} --full --limit 200`);
|
||||
return {
|
||||
ok: response.ok === true,
|
||||
command: "platform-infra observability trace",
|
||||
contentType: "text/plain",
|
||||
renderedText: `${lines.join("\n")}\n`,
|
||||
};
|
||||
}
|
||||
|
||||
function recordValue(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function arrayValue(value: unknown): unknown[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function textValue(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (typeof value === "string") return value;
|
||||
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
||||
return "";
|
||||
}
|
||||
|
||||
function pad(value: string, width: number): string {
|
||||
return value.length >= width ? value : `${value}${" ".repeat(width - value.length)}`;
|
||||
}
|
||||
|
||||
function truncate(value: string, width: number): string {
|
||||
if (value.length <= width) return value;
|
||||
if (width <= 1) return value.slice(0, width);
|
||||
if (width <= 3) return value.slice(0, width);
|
||||
return `${value.slice(0, width - 3)}...`;
|
||||
}
|
||||
|
||||
function renderManifest(observability: ObservabilityConfig, target: ObservabilityTarget): string {
|
||||
@@ -2416,6 +2635,101 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error
|
||||
candidates.sort(key=lambda item: item.get("confidence", 0), reverse=True)
|
||||
return candidates
|
||||
|
||||
def score_trace_candidate(trace_id, meta, index):
|
||||
trace_path = TRACE_PATH_TEMPLATE.replace("{{traceId}}", trace_id)
|
||||
trace_proxy_path = TRACE_PROXY_PREFIX + trace_path
|
||||
trace_rc, trace_body, trace_err = run_kubectl(trace_proxy_path, timeout=5)
|
||||
trace_parsed = parse_json(trace_body)
|
||||
score = 0
|
||||
reasons = []
|
||||
services = []
|
||||
span_count = 0
|
||||
error_span_count = 0
|
||||
business_trace_ids = []
|
||||
root_name = None
|
||||
fetch_ok = trace_rc == 0 and isinstance(trace_parsed, dict)
|
||||
if not fetch_ok:
|
||||
return {
|
||||
"traceId": trace_id,
|
||||
"index": index,
|
||||
"score": -100,
|
||||
"reasons": ["trace_fetch_failed"],
|
||||
"services": services,
|
||||
"spanCount": span_count,
|
||||
"errorSpanCount": error_span_count,
|
||||
"businessTraceIds": business_trace_ids,
|
||||
"rootName": root_name,
|
||||
"fetchOk": False,
|
||||
"selectedMeta": compact_meta(meta),
|
||||
"stderrTail": (trace_err or "")[-1000:],
|
||||
}
|
||||
flat = flatten_trace(trace_parsed)
|
||||
spans = flat.get("spans", [])
|
||||
services = flat.get("services", [])
|
||||
business_trace_ids = flat.get("businessTraceIds", [])
|
||||
error_span_count = len(flat.get("errorSpans", []))
|
||||
span_count = len(spans)
|
||||
if spans:
|
||||
root_name = spans[0].get("name")
|
||||
service_set = set(str(service) for service in services)
|
||||
span_names = [str(item.get("name") or "") for item in spans]
|
||||
attr_fragments = []
|
||||
for item in spans:
|
||||
attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
|
||||
for key in ("runId", "commandId", "runnerJobId", "terminalStatus", "eventType", "failureKind", "traceId"):
|
||||
value = attrs.get(key)
|
||||
if value not in (None, ""):
|
||||
attr_fragments.append("%s=%s" % (key, value))
|
||||
haystack = " ".join(span_names + attr_fragments).lower()
|
||||
if BUSINESS_TRACE_ID and BUSINESS_TRACE_ID in business_trace_ids:
|
||||
score += 20
|
||||
reasons.append("business_trace_id_matched")
|
||||
if "hwlab-cloud-api" in service_set:
|
||||
score += 20
|
||||
reasons.append("hwlab_cloud_api_reached")
|
||||
if "agentrun-manager" in service_set:
|
||||
score += 80
|
||||
reasons.append("agentrun_manager_reached")
|
||||
if "agentrun-runner" in service_set:
|
||||
score += 120
|
||||
reasons.append("agentrun_runner_reached")
|
||||
if "codex_stdio" in haystack:
|
||||
score += 80
|
||||
reasons.append("codex_stdio_spans_present")
|
||||
if "runner_terminal" in haystack or "terminalstatus=" in haystack:
|
||||
score += 60
|
||||
reasons.append("runner_terminal_present")
|
||||
if "runid=" in haystack:
|
||||
score += 30
|
||||
reasons.append("run_id_present")
|
||||
if "commandid=" in haystack:
|
||||
score += 30
|
||||
reasons.append("command_id_present")
|
||||
if error_span_count > 0:
|
||||
score += min(30, error_span_count * 5)
|
||||
reasons.append("error_spans_present")
|
||||
if span_count > 1:
|
||||
score += min(40, span_count)
|
||||
if service_set == {"hwlab-cloud-api"} and span_count <= 2 and any(name.startswith("GET /v1/workbench/events") for name in span_names):
|
||||
score -= 120
|
||||
reasons.append("penalized_workbench_events_sse_only")
|
||||
if any(name.startswith("GET /v1/workbench/events") for name in span_names):
|
||||
score -= 20
|
||||
reasons.append("workbench_events_candidate")
|
||||
return {
|
||||
"traceId": trace_id,
|
||||
"index": index,
|
||||
"score": score,
|
||||
"reasons": reasons,
|
||||
"services": services,
|
||||
"spanCount": span_count,
|
||||
"errorSpanCount": error_span_count,
|
||||
"businessTraceIds": business_trace_ids,
|
||||
"rootName": root_name,
|
||||
"fetchOk": True,
|
||||
"selectedMeta": compact_meta(meta),
|
||||
}
|
||||
|
||||
def resolve_trace():
|
||||
if TRACE_ID is not None:
|
||||
return TRACE_ID, {
|
||||
@@ -2433,14 +2747,36 @@ def resolve_trace():
|
||||
search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15)
|
||||
search_parsed = parse_json(search_body)
|
||||
metas = extract_traces(search_parsed)
|
||||
selected = None
|
||||
selected_trace_id = None
|
||||
for meta in metas:
|
||||
raw_candidates = []
|
||||
for index, meta in enumerate(metas):
|
||||
trace_id = trace_id_from_meta(meta)
|
||||
if trace_id:
|
||||
selected = meta
|
||||
selected_trace_id = trace_id
|
||||
raw_candidates.append((index, trace_id, meta))
|
||||
candidate_probe_limit = 60
|
||||
probe_deadline_seconds = 45
|
||||
probe_deadline = time.time() + probe_deadline_seconds
|
||||
candidate_probe_stopped = None
|
||||
candidate_scores = []
|
||||
for index, trace_id, meta in raw_candidates[:candidate_probe_limit]:
|
||||
if time.time() > probe_deadline:
|
||||
candidate_probe_stopped = "deadline"
|
||||
break
|
||||
scored = score_trace_candidate(trace_id, meta, index)
|
||||
candidate_scores.append(scored)
|
||||
if scored.get("score", 0) >= 260 and len(candidate_scores) >= 12:
|
||||
candidate_probe_stopped = "strong-candidate-found"
|
||||
break
|
||||
candidate_scores.sort(key=lambda item: (item.get("score", -9999), item.get("spanCount", 0), -item.get("index", 0)), reverse=True)
|
||||
selected_score = candidate_scores[0] if candidate_scores else None
|
||||
selected_trace_id = selected_score.get("traceId") if isinstance(selected_score, dict) else None
|
||||
selected = selected_score.get("selectedMeta") if isinstance(selected_score, dict) else None
|
||||
selection_warning = None
|
||||
if selected_score is not None:
|
||||
selected_services = set(str(service) for service in selected_score.get("services", []))
|
||||
if "agentrun-runner" not in selected_services:
|
||||
selection_warning = "no candidate contained agentrun-runner; selected best available trace"
|
||||
if selected_score.get("score", 0) < 0:
|
||||
selection_warning = "candidate fetch/scoring did not find a strong trace; selected best available trace"
|
||||
mapping = {
|
||||
"mode": "business-trace-id",
|
||||
"businessTraceId": BUSINESS_TRACE_ID,
|
||||
@@ -2450,7 +2786,21 @@ def resolve_trace():
|
||||
"searchOk": search_rc == 0,
|
||||
"searchParseOk": isinstance(search_parsed, dict),
|
||||
"candidateTraceCount": len(metas),
|
||||
"selectedMeta": compact_meta(selected),
|
||||
"candidateProbeLimit": candidate_probe_limit,
|
||||
"candidateProbeDeadlineSeconds": probe_deadline_seconds,
|
||||
"candidateProbeStopped": candidate_probe_stopped,
|
||||
"probedCandidateCount": len(candidate_scores),
|
||||
"selectedMeta": selected,
|
||||
"selection": {
|
||||
"mode": "scored-candidates",
|
||||
"score": selected_score.get("score") if isinstance(selected_score, dict) else None,
|
||||
"reasons": selected_score.get("reasons") if isinstance(selected_score, dict) else [],
|
||||
"rootName": selected_score.get("rootName") if isinstance(selected_score, dict) else None,
|
||||
"services": selected_score.get("services") if isinstance(selected_score, dict) else [],
|
||||
"spanCount": selected_score.get("spanCount") if isinstance(selected_score, dict) else None,
|
||||
},
|
||||
"selectionWarning": selection_warning,
|
||||
"candidateScores": candidate_scores[:12],
|
||||
"searchStderrTail": (search_err or "")[-2000:],
|
||||
}
|
||||
if RAW:
|
||||
|
||||
Reference in New Issue
Block a user