mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: diagnose code agent by agentrun ids

Browse Source
This commit is contained in:
Codex
2026-06-26 22:15:12 +00:00
parent 5ddb63ef37
commit 8aae34f610
7 changed files with 191 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-observability/diagnose-code-agent-script.ts
+126 -9
View File
@@ -746,14 +746,75 @@ PY
`;
}
function cliArg(value: string): string {
return /^[A-Za-z0-9_./:=@+-]+$/u.test(value) ? value : shQuote(value);
}
function diagnoseSearchFilters(options: DiagnoseCodeAgentOptions): string[] {
const filters: string[] = [];
if (options.businessTraceId !== null) filters.push(`.traceId = ${JSON.stringify(options.businessTraceId)}`);
if (options.runId !== null) filters.push(`.runId = ${JSON.stringify(options.runId)}`);
if (options.commandId !== null) filters.push(`.commandId = ${JSON.stringify(options.commandId)}`);
if (options.runnerJobId !== null) filters.push(`.runnerJobId = ${JSON.stringify(options.runnerJobId)}`);
return filters;
}
function diagnoseSearchMode(options: DiagnoseCodeAgentOptions): string {
if (options.businessTraceId !== null && options.runId === null && options.commandId === null && options.runnerJobId === null) return "business-trace-id";
if (options.traceId !== null) return "trace-id";
return "trace-attribute-query";
}
function buildDiagnoseCodeAgentCommand(target: ObservabilityTarget, options: DiagnoseCodeAgentOptions, full: boolean): string {
const parts = ["bun", "scripts/cli.ts", "platform-infra", "observability", "diagnose-code-agent", "--target", target.id];
if (options.businessTraceId !== null) parts.push("--business-trace-id", options.businessTraceId);
if (options.traceId !== null) parts.push("--trace-id", options.traceId);
if (options.runId !== null) parts.push("--run-id", options.runId);
if (options.commandId !== null) parts.push("--command-id", options.commandId);
if (options.runnerJobId !== null) parts.push("--runner-job-id", options.runnerJobId);
parts.push("--lookback-minutes", String(options.lookbackMinutes), "--candidate-limit", String(options.candidateLimit), "--limit", String(options.limit));
if (full) parts.push("--full");
return parts.map(cliArg).join(" ");
}
function buildDiagnoseSearchCommand(target: ObservabilityTarget, tempoQuery: string | null): string {
const query = tempoQuery ?? "{ .traceId = \"<trc_...>\" }";
return [
"bun",
"scripts/cli.ts",
"platform-infra",
"observability",
"search",
"--target",
target.id,
"--query",
query,
"--lookback-minutes",
"10080",
"--candidate-limit",
"500",
"--limit",
"10",
].map(cliArg).join(" ");
}
export function diagnoseCodeAgentScript(observability: ObservabilityConfig, target: ObservabilityTarget, searchPath: string | null, options: DiagnoseCodeAgentOptions): string {
const proxyPrefix = `/api/v1/namespaces/${target.namespace}/services/http:${observability.traceBackend.serviceName}:http/proxy`;
const searchProxyPath = searchPath === null ? null : `${proxyPrefix}${searchPath}`;
const traceIdLiteral = options.traceId === null ? "None" : JSON.stringify(options.traceId);
const businessTraceIdLiteral = options.businessTraceId === null ? "None" : JSON.stringify(options.businessTraceId);
const runIdLiteral = options.runId === null ? "None" : JSON.stringify(options.runId);
const commandIdLiteral = options.commandId === null ? "None" : JSON.stringify(options.commandId);
const runnerJobIdLiteral = options.runnerJobId === null ? "None" : JSON.stringify(options.runnerJobId);
const searchPathLiteral = searchPath === null ? "None" : JSON.stringify(searchPath);
const searchProxyPathLiteral = searchProxyPath === null ? "None" : JSON.stringify(searchProxyPath);
const businessTraceIdForNext = options.businessTraceId ?? "<trc_...>";
const tempoQuery = (() => {
const filters = diagnoseSearchFilters(options);
return filters.length > 0 ? `{ ${filters.join(" && ")} }` : null;
})();
const searchMode = diagnoseSearchMode(options);
const diagnoseFullCommand = buildDiagnoseCodeAgentCommand(target, options, true);
const searchCommand = buildDiagnoseSearchCommand(target, tempoQuery);
return `
set -u
python3 - <<'PY'
@@ -761,11 +822,18 @@ import collections, json, re, subprocess, time, urllib.parse
BUSINESS_TRACE_ID = ${businessTraceIdLiteral}
TRACE_ID = ${traceIdLiteral}
RUN_ID = ${runIdLiteral}
COMMAND_ID = ${commandIdLiteral}
RUNNER_JOB_ID = ${runnerJobIdLiteral}
TARGET_ID = ${JSON.stringify(target.id)}
SEARCH_MODE = ${JSON.stringify(searchMode)}
SEARCH_QUERY = ${tempoQuery === null ? "None" : JSON.stringify(tempoQuery)}
SEARCH_PATH = ${searchPathLiteral}
SEARCH_PROXY_PATH = ${searchProxyPathLiteral}
TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)}
TRACE_PATH_TEMPLATE = ${JSON.stringify(observability.probes.traceQueryPathTemplate)}
DIAGNOSE_FULL_COMMAND = ${JSON.stringify(diagnoseFullCommand)}
SEARCH_COMMAND = ${JSON.stringify(searchCommand)}
FULL = ${options.full ? "True" : "False"}
RAW = ${options.raw ? "True" : "False"}
LIMIT = ${options.limit}
@@ -982,6 +1050,28 @@ def classify_terminal_status(status, spans):
return "running"
return "unknown"
def preferred_failure_kind(values):
kinds = []
for value in values:
if value in (None, ""):
continue
text = str(value).strip()
if not text:
continue
if text not in kinds:
kinds.append(text)
if not kinds:
return None
for kind in kinds:
lowered = kind.lower()
if lowered.startswith("provider-") or lowered.startswith("provider_") or "provider-auth" in lowered:
return kind
for kind in kinds:
lowered = kind.lower()
if lowered not in ("infra-failed", "infra_failed", "unknown"):
return kind
return kinds[-1]
def agentrun_authority_summary(identity):
run_id = identity.get("runId") if isinstance(identity, dict) else None
command_id = identity.get("commandId") if isinstance(identity, dict) else None
@@ -1382,7 +1472,7 @@ def http_status_summary(spans):
def agentrun_summary(spans, authority=None):
terminal_spans = []
latest_seq = None
failure_kind = None
failure_kinds = []
failure_message = None
terminal_category = None
seq_keys = ("maxSeq", "traceLastSeq", "endSeq", "seq", "eventSeq", "sourceSeq", "sourceLatestSeq", "latestSeq", "lastSeq")
@@ -1401,7 +1491,7 @@ def agentrun_summary(spans, authority=None):
if not derived and name.startswith("runner_terminal."):
derived = name.split(".", 1)[1]
if attrs.get("failureKind") not in (None, ""):
failure_kind = attrs.get("failureKind")
failure_kinds.append(attrs.get("failureKind"))
if attrs.get("failureMessage") not in (None, ""):
failure_message = attrs.get("failureMessage")
if attrs.get("terminalCategory") not in (None, ""):
@@ -1423,12 +1513,15 @@ def agentrun_summary(spans, authority=None):
if isinstance(authority, dict) and authority.get("terminalStatus") not in (None, ""):
terminal_status = authority.get("terminalStatus")
terminal_event_type = authority.get("terminalEventType") or terminal_event_type
failure_kind = authority.get("failureKind") or failure_kind
if authority.get("failureKind") not in (None, ""):
failure_kinds.append(authority.get("failureKind"))
terminal_source = "agentrun-authority"
failure_kind = preferred_failure_kind(failure_kinds)
runner_classification = classify_terminal_status(terminal_status, spans)
return {
"terminalStatus": terminal_status,
"failureKind": failure_kind,
"observedFailureKinds": failure_kinds,
"failureMessage": failure_message,
"terminalCategory": terminal_category,
"latestSeq": latest_seq,
@@ -1562,6 +1655,20 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error
"summary": "HWLAB projection stale is suspected because AgentRun is terminal but HWLAB read-model evidence is incomplete.",
"evidence": lag_summary.get("reasons", []),
})
failure_kind = str(agentrun.get("failureKind") or "")
if failure_kind == "provider-auth-failed":
candidates.append({
"code": "provider_auth_failed",
"label": "provider auth failed",
"confidence": 0.96,
"summary": "Provider credential refresh/auth failed before the manager reconciler closed the command; fix provider auth before retrying this Code Agent path.",
"evidence": {
"terminalStatus": agentrun.get("terminalStatus"),
"failureKind": agentrun.get("failureKind"),
"observedFailureKinds": agentrun.get("observedFailureKinds"),
"terminalCategory": agentrun.get("terminalCategory"),
},
})
if agentrun.get("terminalStatus") in ("failed", "error", "timeout", "blocked", "cancelled"):
candidates.append({
"code": "agentrun_terminal_failed",
@@ -1712,7 +1819,11 @@ def resolve_trace():
return TRACE_ID, {
"mode": "trace-id",
"businessTraceId": BUSINESS_TRACE_ID,
"runId": RUN_ID,
"commandId": COMMAND_ID,
"runnerJobId": RUNNER_JOB_ID,
"otelTraceId": TRACE_ID,
"searchQuery": SEARCH_QUERY,
"searchPath": None,
"searchOk": None,
"searchParseOk": None,
@@ -1765,9 +1876,13 @@ def resolve_trace():
if selected_low_confidence and not selected_complete:
selected_rejected_reason = "best candidate is only a single-span Workbench GET/SSE trace; no high-confidence Code Agent trace was found in the scanned window"
mapping = {
"mode": "business-trace-id",
"mode": SEARCH_MODE,
"businessTraceId": BUSINESS_TRACE_ID,
"runId": RUN_ID,
"commandId": COMMAND_ID,
"runnerJobId": RUNNER_JOB_ID,
"otelTraceId": selected_trace_id,
"searchQuery": SEARCH_QUERY,
"searchPath": SEARCH_PATH,
"searchProxyPath": SEARCH_PROXY_PATH,
"searchOk": search_rc == 0,
@@ -1787,7 +1902,7 @@ def resolve_trace():
if RAW:
mapping["rawSearchBody"] = search_parsed if isinstance(search_parsed, dict) else search_body[-12000:]
if selected_trace_id is None:
return None, mapping, "no OTel trace found for business trace"
return None, mapping, "no OTel trace found for diagnosis query"
if selected_rejected_reason is not None:
return None, mapping, selected_rejected_reason
return selected_trace_id, mapping, None
@@ -1803,8 +1918,8 @@ if resolved_trace_id is None:
"facts": [],
},
"next": {
"expandWindow": "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${target.id} --business-trace-id ${businessTraceIdForNext} --lookback-minutes 10080 --candidate-limit 500 --full",
"search": "bun scripts/cli.ts platform-infra observability search --target ${target.id} --query '{ .traceId = \\"${businessTraceIdForNext}\\" }' --lookback-minutes 10080 --candidate-limit 500 --limit 10",
"expandWindow": DIAGNOSE_FULL_COMMAND,
"search": SEARCH_COMMAND,
},
}
print(json.dumps(payload, ensure_ascii=False, indent=2))
@@ -1907,6 +2022,8 @@ payload = {
"agentrun": {
"terminalStatus": agentrun.get("terminalStatus"),
"terminalSource": agentrun.get("terminalSource"),
"failureKind": agentrun.get("failureKind"),
"observedFailureKinds": agentrun.get("observedFailureKinds"),
"latestSeq": agentrun.get("latestSeq"),
"terminalEventType": agentrun.get("terminalEventType"),
"runnerProviderClassification": agentrun.get("runnerProviderClassification"),
@@ -1931,7 +2048,7 @@ payload = {
"spanNameCounts": flat["spanNameCounts"][:12],
"evidence": evidence,
"next": {
"diagnoseFull": "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${target.id} --business-trace-id ${businessTraceIdForNext} --full",
"diagnoseFull": DIAGNOSE_FULL_COMMAND,
"traceSummary": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % resolved_trace_id,
"traceReads": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep trace_events_read --limit 20 --full" % resolved_trace_id,
"turnStatus": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep turn_status_read --limit 20 --full" % resolved_trace_id,