From 8aae34f6109cace0c39c9bbf985205fdd5091048 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Jun 2026 22:15:12 +0000 Subject: [PATCH] fix: diagnose code agent by agentrun ids --- .../apply-status-scripts.ts | 7 + .../diagnose-code-agent-script.ts | 135 ++++++++++++++++-- .../platform-infra-observability/manifest.ts | 3 + .../platform-infra-observability/options.ts | 28 +++- .../platform-infra-observability/render.ts | 28 +++- .../src/platform-infra-observability/types.ts | 3 + scripts/src/platform-infra/entry.ts | 1 + 7 files changed, 191 insertions(+), 14 deletions(-) diff --git a/scripts/src/platform-infra-observability/apply-status-scripts.ts b/scripts/src/platform-infra-observability/apply-status-scripts.ts index fa83f15d..5883c5bf 100644 --- a/scripts/src/platform-infra-observability/apply-status-scripts.ts +++ b/scripts/src/platform-infra-observability/apply-status-scripts.ts @@ -61,7 +61,11 @@ export function compactDiagnoseCodeAgentResult(value: unknown): Record terminalSource: agentrun.terminalSource ?? null, latestSeq: agentrun.latestSeq ?? null, terminalEventType: agentrun.terminalEventType ?? null, + failureKind: agentrun.failureKind ?? null, + observedFailureKinds: limitArray(agentrun.observedFailureKinds, 5), runnerProviderClassification: agentrun.runnerProviderClassification ?? null, authority: compactAgentRunAuthority(agentrun.authority), }; @@ -166,6 +172,7 @@ export function compactAgentRunAuthority(value: unknown): Record\" }"; + return [ + "bun", + "scripts/cli.ts", + "platform-infra", + "observability", + "search", + "--target", + target.id, + "--query", + query, + "--lookback-minutes", + "10080", + "--candidate-limit", + "500", + "--limit", + "10", + ].map(cliArg).join(" "); +} + export function diagnoseCodeAgentScript(observability: ObservabilityConfig, target: ObservabilityTarget, searchPath: string | null, options: DiagnoseCodeAgentOptions): string { const proxyPrefix = `/api/v1/namespaces/${target.namespace}/services/http:${observability.traceBackend.serviceName}:http/proxy`; const searchProxyPath = searchPath === null ? null : `${proxyPrefix}${searchPath}`; const traceIdLiteral = options.traceId === null ? "None" : JSON.stringify(options.traceId); const businessTraceIdLiteral = options.businessTraceId === null ? "None" : JSON.stringify(options.businessTraceId); + const runIdLiteral = options.runId === null ? "None" : JSON.stringify(options.runId); + const commandIdLiteral = options.commandId === null ? "None" : JSON.stringify(options.commandId); + const runnerJobIdLiteral = options.runnerJobId === null ? "None" : JSON.stringify(options.runnerJobId); const searchPathLiteral = searchPath === null ? "None" : JSON.stringify(searchPath); const searchProxyPathLiteral = searchProxyPath === null ? "None" : JSON.stringify(searchProxyPath); - const businessTraceIdForNext = options.businessTraceId ?? ""; + const tempoQuery = (() => { + const filters = diagnoseSearchFilters(options); + return filters.length > 0 ? `{ ${filters.join(" && ")} }` : null; + })(); + const searchMode = diagnoseSearchMode(options); + const diagnoseFullCommand = buildDiagnoseCodeAgentCommand(target, options, true); + const searchCommand = buildDiagnoseSearchCommand(target, tempoQuery); return ` set -u python3 - <<'PY' @@ -761,11 +822,18 @@ import collections, json, re, subprocess, time, urllib.parse BUSINESS_TRACE_ID = ${businessTraceIdLiteral} TRACE_ID = ${traceIdLiteral} +RUN_ID = ${runIdLiteral} +COMMAND_ID = ${commandIdLiteral} +RUNNER_JOB_ID = ${runnerJobIdLiteral} TARGET_ID = ${JSON.stringify(target.id)} +SEARCH_MODE = ${JSON.stringify(searchMode)} +SEARCH_QUERY = ${tempoQuery === null ? "None" : JSON.stringify(tempoQuery)} SEARCH_PATH = ${searchPathLiteral} SEARCH_PROXY_PATH = ${searchProxyPathLiteral} TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)} TRACE_PATH_TEMPLATE = ${JSON.stringify(observability.probes.traceQueryPathTemplate)} +DIAGNOSE_FULL_COMMAND = ${JSON.stringify(diagnoseFullCommand)} +SEARCH_COMMAND = ${JSON.stringify(searchCommand)} FULL = ${options.full ? "True" : "False"} RAW = ${options.raw ? "True" : "False"} LIMIT = ${options.limit} @@ -982,6 +1050,28 @@ def classify_terminal_status(status, spans): return "running" return "unknown" +def preferred_failure_kind(values): + kinds = [] + for value in values: + if value in (None, ""): + continue + text = str(value).strip() + if not text: + continue + if text not in kinds: + kinds.append(text) + if not kinds: + return None + for kind in kinds: + lowered = kind.lower() + if lowered.startswith("provider-") or lowered.startswith("provider_") or "provider-auth" in lowered: + return kind + for kind in kinds: + lowered = kind.lower() + if lowered not in ("infra-failed", "infra_failed", "unknown"): + return kind + return kinds[-1] + def agentrun_authority_summary(identity): run_id = identity.get("runId") if isinstance(identity, dict) else None command_id = identity.get("commandId") if isinstance(identity, dict) else None @@ -1382,7 +1472,7 @@ def http_status_summary(spans): def agentrun_summary(spans, authority=None): terminal_spans = [] latest_seq = None - failure_kind = None + failure_kinds = [] failure_message = None terminal_category = None seq_keys = ("maxSeq", "traceLastSeq", "endSeq", "seq", "eventSeq", "sourceSeq", "sourceLatestSeq", "latestSeq", "lastSeq") @@ -1401,7 +1491,7 @@ def agentrun_summary(spans, authority=None): if not derived and name.startswith("runner_terminal."): derived = name.split(".", 1)[1] if attrs.get("failureKind") not in (None, ""): - failure_kind = attrs.get("failureKind") + failure_kinds.append(attrs.get("failureKind")) if attrs.get("failureMessage") not in (None, ""): failure_message = attrs.get("failureMessage") if attrs.get("terminalCategory") not in (None, ""): @@ -1423,12 +1513,15 @@ def agentrun_summary(spans, authority=None): if isinstance(authority, dict) and authority.get("terminalStatus") not in (None, ""): terminal_status = authority.get("terminalStatus") terminal_event_type = authority.get("terminalEventType") or terminal_event_type - failure_kind = authority.get("failureKind") or failure_kind + if authority.get("failureKind") not in (None, ""): + failure_kinds.append(authority.get("failureKind")) terminal_source = "agentrun-authority" + failure_kind = preferred_failure_kind(failure_kinds) runner_classification = classify_terminal_status(terminal_status, spans) return { "terminalStatus": terminal_status, "failureKind": failure_kind, + "observedFailureKinds": failure_kinds, "failureMessage": failure_message, "terminalCategory": terminal_category, "latestSeq": latest_seq, @@ -1562,6 +1655,20 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error "summary": "HWLAB projection stale is suspected because AgentRun is terminal but HWLAB read-model evidence is incomplete.", "evidence": lag_summary.get("reasons", []), }) + failure_kind = str(agentrun.get("failureKind") or "") + if failure_kind == "provider-auth-failed": + candidates.append({ + "code": "provider_auth_failed", + "label": "provider auth failed", + "confidence": 0.96, + "summary": "Provider credential refresh/auth failed before the manager reconciler closed the command; fix provider auth before retrying this Code Agent path.", + "evidence": { + "terminalStatus": agentrun.get("terminalStatus"), + "failureKind": agentrun.get("failureKind"), + "observedFailureKinds": agentrun.get("observedFailureKinds"), + "terminalCategory": agentrun.get("terminalCategory"), + }, + }) if agentrun.get("terminalStatus") in ("failed", "error", "timeout", "blocked", "cancelled"): candidates.append({ "code": "agentrun_terminal_failed", @@ -1712,7 +1819,11 @@ def resolve_trace(): return TRACE_ID, { "mode": "trace-id", "businessTraceId": BUSINESS_TRACE_ID, + "runId": RUN_ID, + "commandId": COMMAND_ID, + "runnerJobId": RUNNER_JOB_ID, "otelTraceId": TRACE_ID, + "searchQuery": SEARCH_QUERY, "searchPath": None, "searchOk": None, "searchParseOk": None, @@ -1765,9 +1876,13 @@ def resolve_trace(): if selected_low_confidence and not selected_complete: selected_rejected_reason = "best candidate is only a single-span Workbench GET/SSE trace; no high-confidence Code Agent trace was found in the scanned window" mapping = { - "mode": "business-trace-id", + "mode": SEARCH_MODE, "businessTraceId": BUSINESS_TRACE_ID, + "runId": RUN_ID, + "commandId": COMMAND_ID, + "runnerJobId": RUNNER_JOB_ID, "otelTraceId": selected_trace_id, + "searchQuery": SEARCH_QUERY, "searchPath": SEARCH_PATH, "searchProxyPath": SEARCH_PROXY_PATH, "searchOk": search_rc == 0, @@ -1787,7 +1902,7 @@ def resolve_trace(): if RAW: mapping["rawSearchBody"] = search_parsed if isinstance(search_parsed, dict) else search_body[-12000:] if selected_trace_id is None: - return None, mapping, "no OTel trace found for business trace" + return None, mapping, "no OTel trace found for diagnosis query" if selected_rejected_reason is not None: return None, mapping, selected_rejected_reason return selected_trace_id, mapping, None @@ -1803,8 +1918,8 @@ if resolved_trace_id is None: "facts": [], }, "next": { - "expandWindow": "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${target.id} --business-trace-id ${businessTraceIdForNext} --lookback-minutes 10080 --candidate-limit 500 --full", - "search": "bun scripts/cli.ts platform-infra observability search --target ${target.id} --query '{ .traceId = \\"${businessTraceIdForNext}\\" }' --lookback-minutes 10080 --candidate-limit 500 --limit 10", + "expandWindow": DIAGNOSE_FULL_COMMAND, + "search": SEARCH_COMMAND, }, } print(json.dumps(payload, ensure_ascii=False, indent=2)) @@ -1907,6 +2022,8 @@ payload = { "agentrun": { "terminalStatus": agentrun.get("terminalStatus"), "terminalSource": agentrun.get("terminalSource"), + "failureKind": agentrun.get("failureKind"), + "observedFailureKinds": agentrun.get("observedFailureKinds"), "latestSeq": agentrun.get("latestSeq"), "terminalEventType": agentrun.get("terminalEventType"), "runnerProviderClassification": agentrun.get("runnerProviderClassification"), @@ -1931,7 +2048,7 @@ payload = { "spanNameCounts": flat["spanNameCounts"][:12], "evidence": evidence, "next": { - "diagnoseFull": "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${target.id} --business-trace-id ${businessTraceIdForNext} --full", + "diagnoseFull": DIAGNOSE_FULL_COMMAND, "traceSummary": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --limit 80" % resolved_trace_id, "traceReads": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep trace_events_read --limit 20 --full" % resolved_trace_id, "turnStatus": "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --grep turn_status_read --limit 20 --full" % resolved_trace_id, diff --git a/scripts/src/platform-infra-observability/manifest.ts b/scripts/src/platform-infra-observability/manifest.ts index 680c39f0..d9562330 100644 --- a/scripts/src/platform-infra-observability/manifest.ts +++ b/scripts/src/platform-infra-observability/manifest.ts @@ -207,6 +207,9 @@ export function buildDiagnoseCommand(target: ObservabilityTarget, options: Diagn const parts = ["bun", "scripts/cli.ts", "platform-infra", "observability", "diagnose-code-agent", "--target", target.id]; if (options.businessTraceId !== null) parts.push("--business-trace-id", options.businessTraceId); if (options.traceId !== null) parts.push("--trace-id", options.traceId); + if (options.runId !== null) parts.push("--run-id", options.runId); + if (options.commandId !== null) parts.push("--command-id", options.commandId); + if (options.runnerJobId !== null) parts.push("--runner-job-id", options.runnerJobId); parts.push("--lookback-minutes", String(options.lookbackMinutes), "--candidate-limit", String(options.candidateLimit), "--limit", String(options.limit)); if (full) parts.push("--full"); return parts.map(cliArg).join(" "); diff --git a/scripts/src/platform-infra-observability/options.ts b/scripts/src/platform-infra-observability/options.ts index 0e0376cb..656c689e 100644 --- a/scripts/src/platform-infra-observability/options.ts +++ b/scripts/src/platform-infra-observability/options.ts @@ -39,6 +39,7 @@ export function observabilityHelp(): Record { "bun scripts/cli.ts platform-infra observability search --target D601 --grep 'no rollout found' [--lookback-minutes 360] [--candidate-limit 80] [--limit 20] [--full|--raw]", "bun scripts/cli.ts platform-infra observability search --target D601 --path /v1/workbench/sessions --status 502 [--lookback-minutes 120] [--full|--raw]", "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target D601 --business-trace-id [--full|--raw]", + "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target D601 --run-id [--command-id ] [--runner-job-id ] [--full|--raw]", ], boundary: "Prometheus remains the metrics source; this command owns only platform-infra OTel Collector, trace backend readiness, and trace lookup.", }; @@ -217,6 +218,9 @@ export function parseDiagnoseCodeAgentOptions(args: string[]): DiagnoseCodeAgent const commonArgs: string[] = []; let businessTraceId: string | null = null; let traceId: string | null = null; + let runId: string | null = null; + let commandId: string | null = null; + let runnerJobId: string | null = null; let limit = 40; let candidateLimit = 300; let lookbackMinutes = 10080; @@ -234,6 +238,24 @@ export function parseDiagnoseCodeAgentOptions(args: string[]): DiagnoseCodeAgent if (!/^[0-9a-fA-F]{32}$/u.test(value)) throw new Error(`${arg} must be a 32-character OpenTelemetry trace id encoded as hex`); traceId = value.toLowerCase(); index += 1; + } else if (arg === "--run-id" || arg === "--run") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); + if (!/^run_[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`${arg} must look like run_`); + runId = value; + index += 1; + } else if (arg === "--command-id" || arg === "--command") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); + if (!/^cmd_[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`${arg} must look like cmd_`); + commandId = value; + index += 1; + } else if (arg === "--runner-job-id" || arg === "--runner-job" || arg === "--runnerjob") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); + if (!/^rjob_[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`${arg} must look like rjob_`); + runnerJobId = value; + index += 1; } else if (arg === "--limit") { const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error("--limit requires a value"); @@ -263,6 +285,8 @@ export function parseDiagnoseCodeAgentOptions(args: string[]): DiagnoseCodeAgent } } } - if (businessTraceId === null && traceId === null) throw new Error("observability diagnose-code-agent requires --business-trace-id or --trace-id "); - return { ...parseCommonOptions(commonArgs), businessTraceId, traceId, limit, candidateLimit, lookbackMinutes }; + if (businessTraceId === null && traceId === null && runId === null && commandId === null && runnerJobId === null) { + throw new Error("observability diagnose-code-agent requires --business-trace-id , --trace-id , --run-id , --command-id , or --runner-job-id "); + } + return { ...parseCommonOptions(commonArgs), businessTraceId, traceId, runId, commandId, runnerJobId, limit, candidateLimit, lookbackMinutes }; } diff --git a/scripts/src/platform-infra-observability/render.ts b/scripts/src/platform-infra-observability/render.ts index 207af3e3..cc7b11ad 100644 --- a/scripts/src/platform-infra-observability/render.ts +++ b/scripts/src/platform-infra-observability/render.ts @@ -141,18 +141,34 @@ export function businessTraceIdFromSearchText(value: string | null): string | nu return match?.[0] ?? null; } +function diagnoseTraceSearchFilters(options: DiagnoseCodeAgentOptions): string[] { + const filters: string[] = []; + if (options.businessTraceId !== null) filters.push(`.traceId = ${JSON.stringify(options.businessTraceId)}`); + if (options.runId !== null) filters.push(`.runId = ${JSON.stringify(options.runId)}`); + if (options.commandId !== null) filters.push(`.commandId = ${JSON.stringify(options.commandId)}`); + if (options.runnerJobId !== null) filters.push(`.runnerJobId = ${JSON.stringify(options.runnerJobId)}`); + return filters; +} + +function diagnoseTraceSearchMode(options: DiagnoseCodeAgentOptions): string { + if (options.businessTraceId !== null && options.runId === null && options.commandId === null && options.runnerJobId === null) return "business-trace-id"; + return "trace-attribute-query"; +} + export async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); const endSeconds = Math.floor(Date.now() / 1000); const startSeconds = endSeconds - (options.lookbackMinutes * 60); let searchPath: string | null = null; - if (options.traceId === null && options.businessTraceId !== null) { + const searchFilters = diagnoseTraceSearchFilters(options); + const tempoQuery = searchFilters.length > 0 ? `{ ${searchFilters.join(" && ")} }` : null; + if (options.traceId === null && tempoQuery !== null) { const params = new URLSearchParams({ start: String(startSeconds), end: String(endSeconds), limit: String(options.candidateLimit), - q: `{ .traceId = "${options.businessTraceId}" }`, + q: tempoQuery, }); searchPath = `/api/search?${params.toString()}`; } @@ -164,6 +180,11 @@ export async function diagnoseCodeAgent(config: UniDeskConfig, options: Diagnose service: observability.traceBackend.serviceName, businessTraceId: options.businessTraceId, traceId: options.traceId, + runId: options.runId, + commandId: options.commandId, + runnerJobId: options.runnerJobId, + mode: diagnoseTraceSearchMode(options), + tempoQuery, path: searchPath, lookbackMinutes: options.lookbackMinutes, candidateLimit: options.candidateLimit, @@ -530,6 +551,7 @@ export function renderDiagnoseCodeAgentTable(input: { "", "Identity:", ` businessTraceId=${textValue(mapping?.businessTraceId ?? input.query.businessTraceId)} otelTraceId=${traceId}`, + ` queryMode=${textValue(mapping?.mode ?? input.query.mode)} tempoQuery=${shortenMiddle(textValue(mapping?.searchQuery ?? input.query.tempoQuery), 80)}`, ` runId=${textValue(identity?.runId)} commandId=${textValue(identity?.commandId)} runnerJobId=${textValue(identity?.runnerJobId)} runnerId=${textValue(identity?.runnerId)}`, ` backendProfile=${textValue(identity?.backendProfile)} sourceCommit=${shortenMiddle(textValue(identity?.sourceCommit), 20)}`, "", @@ -541,7 +563,7 @@ export function renderDiagnoseCodeAgentTable(input: { "", "Summary:", ` target=${input.target.id} spanCount=${textValue(input.result.spanCount)} servicePath=${joinValues(input.result.servicePath, 60)}`, - ` agentrun=${textValue(agentrun?.terminalStatus)} source=${textValue(agentrun?.terminalSource)} authority=${shortenEnd(JSON.stringify((asPlainRecord(agentrun?.authority) ?? {})), 120)}`, + ` agentrun=${textValue(agentrun?.terminalStatus)} failureKind=${textValue(agentrun?.failureKind)} source=${textValue(agentrun?.terminalSource)} authority=${shortenEnd(JSON.stringify((asPlainRecord(agentrun?.authority) ?? {})), 120)}`, ` readModel=${shortenEnd(JSON.stringify(input.result.hwlabReadModel ?? null), 140)}`, ]; const next = asPlainRecord(input.result.next); diff --git a/scripts/src/platform-infra-observability/types.ts b/scripts/src/platform-infra-observability/types.ts index d2ced1e8..06c9cc56 100644 --- a/scripts/src/platform-infra-observability/types.ts +++ b/scripts/src/platform-infra-observability/types.ts @@ -157,6 +157,9 @@ export interface SearchOptions extends CommonOptions { export interface DiagnoseCodeAgentOptions extends CommonOptions { businessTraceId: string | null; traceId: string | null; + runId: string | null; + commandId: string | null; + runnerJobId: string | null; limit: number; candidateLimit: number; lookbackMinutes: number; diff --git a/scripts/src/platform-infra/entry.ts b/scripts/src/platform-infra/entry.ts index d375e748..b56264cc 100644 --- a/scripts/src/platform-infra/entry.ts +++ b/scripts/src/platform-infra/entry.ts @@ -359,6 +359,7 @@ export function platformInfraHelp(): unknown { "bun scripts/cli.ts platform-infra observability trace --target D601 --trace-id [--grep text] [--limit 40] [--full|--raw]", "bun scripts/cli.ts platform-infra observability search --target D601 --grep 'no rollout found' [--lookback-minutes 360] [--candidate-limit 80] [--limit 20]", "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target D601 --business-trace-id [--full|--raw]", + "bun scripts/cli.ts platform-infra observability diagnose-code-agent --target D601 --run-id [--command-id ] [--runner-job-id ]", ], description: "Operate YAML-controlled platform-infra services such as Sub2API, LangBot, n8n, WeChat archive workflows and OpenTelemetry tracing. Public services use PK01 Caddy+FRP rather than Kubernetes Ingress, NodePort, or LoadBalancer.", target,