From 327fe836a2cb70c291da71ab8c3e1932b9f2c7e7 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 11 Jul 2026 23:40:41 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20=E9=9A=94=E7=A6=BB=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E4=BB=A3=E7=90=86=E8=AF=8A=E6=96=AD=20trace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apply-status-scripts.ts | 69 ++++++-- .../diagnose-code-agent-script.test.ts | 88 ++++++++++ .../diagnose-code-agent-script.ts | 155 ++++++++++++++++-- .../platform-infra-observability/render.ts | 11 +- 4 files changed, 293 insertions(+), 30 deletions(-) diff --git a/scripts/src/platform-infra-observability/apply-status-scripts.ts b/scripts/src/platform-infra-observability/apply-status-scripts.ts index 442f1e26..c5fb26d9 100644 --- a/scripts/src/platform-infra-observability/apply-status-scripts.ts +++ b/scripts/src/platform-infra-observability/apply-status-scripts.ts @@ -90,6 +90,7 @@ export function compactDiagnoseCodeAgentResult(value: unknown): Record | null { + const diagnostic = asPlainRecord(value); + if (diagnostic === null) return null; + return { + traceId: diagnostic.traceId ?? null, + parentSpanId: diagnostic.parentSpanId ?? null, + traceparent: diagnostic.traceparent ?? null, + operation: diagnostic.operation ?? null, + targetBusinessTraceId: diagnostic.targetBusinessTraceId ?? null, + targetCommandId: diagnostic.targetCommandId ?? null, + expectedTargetOtelTraceId: diagnostic.expectedTargetOtelTraceId ?? null, + contextReady: diagnostic.contextReady ?? null, + headerReadPaths: limitArray(diagnostic.headerReadPaths, 3), + ordinaryReadPaths: limitArray(diagnostic.ordinaryReadPaths, 2), + traceCommand: diagnostic.traceCommand ?? null, + targetTraceCommand: diagnostic.targetTraceCommand ?? null, + linkQueryClue: diagnostic.linkQueryClue ?? null, + }; +} + export function compactDiagnoseIdentity(value: unknown): Record | null { return compactRecord(value, [ "runId", @@ -169,7 +190,10 @@ export function compactAgentRunAuthority(value: unknown): Record { - const attempt = asPlainRecord(item) ?? {}; - return { - name: attempt.name ?? attempt.section ?? attempt.kind ?? null, - ok: attempt.ok ?? null, - status: attempt.status ?? null, - exitCode: attempt.exitCode ?? null, - url: attempt.url ?? null, - error: shortenEnd(textValue(attempt.error), 240), - stderrTail: shortenEnd(textValue(attempt.stderrTail ?? attempt.stderr), 500), - }; - }); +export function compactAgentRunAuthorityAttempts(value: unknown): unknown { + const grouped = asPlainRecord(value); + if (grouped !== null) { + return Object.fromEntries(Object.entries(grouped).slice(0, 4).map(([name, attempts]) => [ + name, + asArray(attempts).slice(0, 3).map((attempt) => compactAgentRunAuthorityAttempt(attempt)), + ])); + } + return asArray(value).slice(0, 3).map((attempt) => compactAgentRunAuthorityAttempt(attempt)); +} + +function compactAgentRunAuthorityAttempt(value: unknown): Record { + const attempt = asPlainRecord(value) ?? {}; + return { + name: attempt.name ?? attempt.section ?? attempt.kind ?? null, + ok: attempt.ok ?? null, + status: attempt.status ?? null, + exitCode: attempt.exitCode ?? attempt.rc ?? null, + method: attempt.method ?? null, + url: attempt.url ?? null, + proxyPath: attempt.proxyPath ?? null, + diagnosticContextApplied: attempt.diagnosticContextApplied ?? null, + fallbackDenied: attempt.fallbackDenied ?? null, + error: shortenEnd(textValue(attempt.error), 240), + stderrTail: shortenEnd(textValue(attempt.stderrTail ?? attempt.stderr), 500), + }; } export function compactDiagnoseCandidate(value: unknown): Record { diff --git a/scripts/src/platform-infra-observability/diagnose-code-agent-script.test.ts b/scripts/src/platform-infra-observability/diagnose-code-agent-script.test.ts index d3f8be1f..27814fe1 100644 --- a/scripts/src/platform-infra-observability/diagnose-code-agent-script.test.ts +++ b/scripts/src/platform-infra-observability/diagnose-code-agent-script.test.ts @@ -27,6 +27,11 @@ describe("diagnose-code-agent warm-session terminal classification", () => { namespaceSource: "provider-decision", providerDecisionNamespace: "agentrun-v02", ok: true, + commandOk: true, + resultOk: true, + runResultOk: true, + eventsOk: true, + diagnosticTransportFailed: false, terminalStatus: "completed", terminalEventSeq: 10, eventCount: 2, @@ -42,6 +47,7 @@ describe("diagnose-code-agent warm-session terminal classification", () => { }); expect(diagnosis.calls).toContain("-n agentrun-v02 exec deploy/agentrun-mgr"); expect(diagnosis.calls).not.toContain("agentrun-v01"); + assertDiagnosticRequestContract(diagnosis); const compact = compactDiagnoseCodeAgentResult(diagnosis.payload); expect((compact.agentrun as Record).authority).toMatchObject({ @@ -57,6 +63,10 @@ describe("diagnose-code-agent warm-session terminal classification", () => { result: compact, }); expect(rendered.renderedText).toContain("authorityQuery namespace=agentrun-v02 source=provider-decision ok=true commandState=completed terminal=completed"); + expect(rendered.renderedText).toContain("Diagnostic trace:"); + expect(rendered.renderedText).toContain(`traceId=${diagnosis.payload.diagnosticTrace.traceId}`); + expect(rendered.renderedText).toContain("expectedLinkTraceId=de11d47d1bab0ca4d3f6259279f3a332"); + expect(rendered.renderedText).toContain(`--target NC01 --trace-id ${diagnosis.payload.diagnosticTrace.traceId} --full`); }); test("reports unknown instead of running when command authority and terminal span are both absent", () => { @@ -73,6 +83,20 @@ describe("diagnose-code-agent warm-session terminal classification", () => { terminalSpanSeen: false, }); expect(diagnosis.payload.rootCauseCandidates[0].code).toBe("observability_gap_authority_query_failed"); + expect(diagnosis.payload.agentrun.authority).toMatchObject({ + diagnosticTransportFailed: true, + diagnosticTransportFailures: ["command-result", "run-result", "events"], + attempts: { + result: [{ diagnosticContextApplied: true, fallbackDenied: "headerless-service-proxy", error: "diagnostic-transport-failed" }], + runResult: [{ diagnosticContextApplied: true, fallbackDenied: "headerless-service-proxy", error: "diagnostic-transport-failed" }], + events: [{ diagnosticContextApplied: true, fallbackDenied: "headerless-service-proxy", error: "diagnostic-transport-failed" }], + }, + }); + expect(diagnosis.payload.agentrun.authority.attempts.command[0].diagnosticContextApplied).toBe(false); + expect(diagnosis.payload.agentrun.authority.attempts.result).toHaveLength(1); + expect(diagnosis.payload.agentrun.authority.attempts.runResult).toHaveLength(1); + expect(diagnosis.payload.agentrun.authority.attempts.events).toHaveLength(1); + assertNoHeaderlessDiagnosticFallback(diagnosis.calls); }); test("keeps a scoped OTel terminal span complete even if the supplemental authority read fails", () => { @@ -90,6 +114,15 @@ describe("diagnose-code-agent warm-session terminal classification", () => { terminalSpanCount: 1, }); }); + + test("uses a fresh diagnostic trace for each repeated diagnosis without unscoped authority reads", () => { + const first = runFixture({ authorityAvailable: true, terminalSpan: false }); + const second = runFixture({ authorityAvailable: true, terminalSpan: false }); + assertDiagnosticRequestContract(first); + assertDiagnosticRequestContract(second); + expect(first.payload.diagnosticTrace.traceId).not.toBe(second.payload.diagnosticTrace.traceId); + expect(first.payload.diagnosticTrace.traceparent).not.toBe(second.payload.diagnosticTrace.traceparent); + }); }); function runFixture(input: { authorityAvailable: boolean; terminalSpan: boolean }): { @@ -102,6 +135,7 @@ function runFixture(input: { authorityAvailable: boolean; terminalSpan: boolean writeFileSync(join(directory, "trace.json"), JSON.stringify(traceFixture(input.terminalSpan))); writeFileSync(join(directory, "command.json"), JSON.stringify({ id: "cmd_second", state: "completed" })); writeFileSync(join(directory, "result.json"), JSON.stringify({ ok: true, commandId: "cmd_second", status: "completed", commandState: "completed", terminalStatus: "completed" })); + writeFileSync(join(directory, "run-result.json"), JSON.stringify({ ok: true, commandId: "cmd_second", commandState: "completed", terminalStatus: "completed" })); writeFileSync(join(directory, "events.json"), JSON.stringify({ items: [ { seq: 10, type: "terminal_status", payload: { commandId: "cmd_second", terminalStatus: "completed" } }, @@ -121,6 +155,7 @@ if [ "$AUTHORITY_AVAILABLE" != "1" ]; then fi case "$*" in *"-n agentrun-v02 exec deploy/agentrun-mgr"*"/commands/cmd_second/result"*) cat "$FIXTURE_DIR/result.json"; exit 0 ;; + *"-n agentrun-v02 exec deploy/agentrun-mgr"*"/runs/run_warm/result?commandId=cmd_second"*) cat "$FIXTURE_DIR/run-result.json"; exit 0 ;; *"-n agentrun-v02 exec deploy/agentrun-mgr"*"/commands/cmd_second"*) cat "$FIXTURE_DIR/command.json"; exit 0 ;; *"-n agentrun-v02 exec deploy/agentrun-mgr"*"/runs/run_warm/events"*) cat "$FIXTURE_DIR/events.json"; exit 0 ;; esac @@ -147,6 +182,59 @@ exit 1 }; } +function assertDiagnosticRequestContract(diagnosis: { payload: Record; calls: string }): void { + const diagnostic = diagnosis.payload.diagnosticTrace; + expect(diagnostic).toMatchObject({ + operation: "unidesk.diagnose-code-agent", + targetBusinessTraceId: "trc_warm_second", + targetCommandId: "cmd_second", + expectedTargetOtelTraceId: "de11d47d1bab0ca4d3f6259279f3a332", + contextReady: true, + headerReadPaths: ["command-result", "run-result", "events"], + ordinaryReadPaths: ["command-detail"], + }); + expect(diagnostic.traceId).toMatch(/^[0-9a-f]{32}$/); + expect(diagnostic.parentSpanId).toMatch(/^[0-9a-f]{16}$/); + expect(diagnostic.traceparent).toBe(`00-${diagnostic.traceId}-${diagnostic.parentSpanId}-01`); + expect(diagnostic.traceCommand).toContain(`--target NC01 --trace-id ${diagnostic.traceId} --full`); + expect(diagnostic.targetTraceCommand).toContain("--target NC01 --trace-id de11d47d1bab0ca4d3f6259279f3a332 --full"); + expect(diagnostic.linkQueryClue).toContain("links for traceId=de11d47d1bab0ca4d3f6259279f3a332"); + + const lines = callLines(diagnosis.calls); + const diagnosticLines = lines.filter((line) => line.includes("x-agentrun-trace-purpose: diagnostic")); + expect(diagnosticLines).toHaveLength(3); + for (const path of [ + "/api/v1/runs/run_warm/commands/cmd_second/result", + "/api/v1/runs/run_warm/result?commandId=cmd_second", + "/api/v1/runs/run_warm/events?afterSeq=0&limit=1000", + ]) { + const line = diagnosticLines.find((candidate) => candidate.includes(path)); + expect(line).toBeDefined(); + expect(line).toContain(`traceparent: ${diagnostic.traceparent}`); + expect(line).toContain("x-agentrun-diagnostic-operation: unidesk.diagnose-code-agent"); + expect(line).toContain("x-agentrun-diagnostic-target-business-trace-id: trc_warm_second"); + expect(line).toContain("x-agentrun-diagnostic-target-command-id: cmd_second"); + } + const traceparents = new Set(diagnosticLines.flatMap((line) => line.match(/00-[0-9a-f]{32}-[0-9a-f]{16}-01/g) ?? [])); + expect([...traceparents]).toEqual([diagnostic.traceparent]); + + const commandDetailLines = lines.filter((line) => line.includes("/api/v1/runs/run_warm/commands/cmd_second") && !line.includes("/commands/cmd_second/result")); + expect(commandDetailLines).toHaveLength(1); + expect(commandDetailLines[0]).not.toContain("x-agentrun-trace-purpose"); + assertNoHeaderlessDiagnosticFallback(diagnosis.calls); +} + +function assertNoHeaderlessDiagnosticFallback(calls: string): void { + const proxyLines = callLines(calls).filter((line) => line.includes("services/http:agentrun-mgr")); + expect(proxyLines.some((line) => line.includes("/commands/cmd_second/result"))).toBe(false); + expect(proxyLines.some((line) => line.includes("/runs/run_warm/result?commandId=cmd_second"))).toBe(false); + expect(proxyLines.some((line) => line.includes("/runs/run_warm/events?afterSeq=0&limit=1000"))).toBe(false); +} + +function callLines(calls: string): string[] { + return calls.split("\n").map((line) => line.trim()).filter(Boolean); +} + function traceFixture(terminalSpan: boolean): Record { const common = [ attribute("traceId", "trc_warm_second"), diff --git a/scripts/src/platform-infra-observability/diagnose-code-agent-script.ts b/scripts/src/platform-infra-observability/diagnose-code-agent-script.ts index e9c55ea8..8c08b532 100644 --- a/scripts/src/platform-infra-observability/diagnose-code-agent-script.ts +++ b/scripts/src/platform-infra-observability/diagnose-code-agent-script.ts @@ -911,7 +911,7 @@ export function diagnoseCodeAgentScript(observability: ObservabilityConfig, targ return ` set -u python3 - <<'PY' -import collections, json, re, subprocess, time, urllib.parse +import collections, json, re, secrets, subprocess, time, urllib.parse BUSINESS_TRACE_ID = ${businessTraceIdLiteral} TRACE_ID = ${traceIdLiteral} @@ -932,6 +932,7 @@ FULL = ${options.full ? "True" : "False"} RAW = ${options.raw ? "True" : "False"} LIMIT = ${options.limit} CANDIDATE_LIMIT = ${options.candidateLimit} +DIAGNOSTIC_OPERATION = "unidesk.diagnose-code-agent" DEADLINE = time.time() + 50 IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", @@ -1077,16 +1078,38 @@ def agentrun_service_proxy_paths(namespace, api_path): "/api/v1/namespaces/%s/services/http:agentrun-mgr:8080/proxy%s" % (namespace, api_path), ] -def run_agentrun_mgr_json(namespace, api_path, timeout=12): +def diagnostic_header_items(diagnostic_context): + if not isinstance(diagnostic_context, dict): + return [] + return [ + ("traceparent", diagnostic_context.get("traceparent")), + ("x-agentrun-trace-purpose", "diagnostic"), + ("x-agentrun-diagnostic-operation", diagnostic_context.get("operation")), + ("x-agentrun-diagnostic-target-business-trace-id", diagnostic_context.get("targetBusinessTraceId")), + ("x-agentrun-diagnostic-target-command-id", diagnostic_context.get("targetCommandId")), + ] + +def run_agentrun_mgr_json(namespace, api_path, timeout=12, diagnostic_context=None): url = "http://127.0.0.1:8080%s" % api_path + diagnostic_headers = [ + (name, str(value)) + for name, value in diagnostic_header_items(diagnostic_context) + if value not in (None, "") + ] script_parts = [] script_parts.append("url=" + shell_quote(url) + "; ") script_parts.append("api_key=\\\"\${AGENTRUN_API_KEY:-}\\\"; ") script_parts.append("if [ -z \\\"$api_key\\\" ]; then api_key=\\\"\${HWLAB_API_KEY:-}\\\"; fi; ") script_parts.append("if command -v wget >/dev/null 2>&1; then ") - script_parts.append("wget -qO- --header=\\\"Authorization: Bearer $api_key\\\" \\\"$url\\\"; ") + script_parts.append("wget -qO- --header=\\\"Authorization: Bearer $api_key\\\" ") + for name, value in diagnostic_headers: + script_parts.append("--header=" + shell_quote(name + ": " + value) + " ") + script_parts.append("\\\"$url\\\"; ") script_parts.append("else ") - script_parts.append("curl -fsS -H \\\"Authorization: Bearer $api_key\\\" \\\"$url\\\"; ") + script_parts.append("curl -fsS -H \\\"Authorization: Bearer $api_key\\\" ") + for name, value in diagnostic_headers: + script_parts.append("-H " + shell_quote(name + ": " + value) + " ") + script_parts.append("\\\"$url\\\"; ") script_parts.append("fi") script = "".join(script_parts) try: @@ -1104,18 +1127,23 @@ def run_agentrun_mgr_json(namespace, api_path, timeout=12): "stderrTail": (proc.stderr or "")[-1000:], "bodyBytes": len((proc.stdout or "").encode("utf-8")), "method": "manager-exec-localhost", + "diagnosticContextApplied": bool(diagnostic_headers), } except subprocess.TimeoutExpired as exc: - return {"ok": False, "rc": 124, "json": None, "stderrTail": "timeout while querying agentrun manager", "bodyBytes": len((exc.stdout or "").encode("utf-8")), "method": "manager-exec-localhost"} + return {"ok": False, "rc": 124, "json": None, "stderrTail": "timeout while querying agentrun manager", "bodyBytes": len((exc.stdout or "").encode("utf-8")), "method": "manager-exec-localhost", "diagnosticContextApplied": bool(diagnostic_headers)} except Exception as exc: - return {"ok": False, "rc": 125, "json": None, "stderrTail": str(exc), "bodyBytes": 0, "method": "manager-exec-localhost"} + return {"ok": False, "rc": 125, "json": None, "stderrTail": str(exc), "bodyBytes": 0, "method": "manager-exec-localhost", "diagnosticContextApplied": bool(diagnostic_headers)} -def get_agentrun_json(namespace, api_path, timeout=10): +def get_agentrun_json(namespace, api_path, timeout=10, diagnostic_context=None): attempts = [] - exec_response = run_agentrun_mgr_json(namespace, api_path, timeout=timeout) + exec_response = run_agentrun_mgr_json(namespace, api_path, timeout=timeout, diagnostic_context=diagnostic_context) attempts.append(exec_response) if exec_response.get("ok"): return exec_response, attempts + if isinstance(diagnostic_context, dict): + exec_response["error"] = "diagnostic-transport-failed" + exec_response["fallbackDenied"] = "headerless-service-proxy" + return exec_response, attempts for proxy_path in agentrun_service_proxy_paths(namespace, api_path): response = run_kubectl_json(proxy_path, timeout=timeout) response["proxyPath"] = proxy_path @@ -1196,7 +1224,14 @@ def preferred_failure_kind(values): return kind return kinds[-1] -def agentrun_authority_summary(identity, provider_decision): +def diagnostic_transport_failure_names(responses): + return [ + name + for name, response in responses + if isinstance(response, dict) and response.get("error") == "diagnostic-transport-failed" + ] + +def agentrun_authority_summary(identity, provider_decision, diagnostic_context): run_id = identity.get("runId") if isinstance(identity, dict) else None command_id = identity.get("commandId") if isinstance(identity, dict) else None if not run_id or not command_id: @@ -1204,12 +1239,18 @@ def agentrun_authority_summary(identity, provider_decision): "queried": False, "reason": "missing-run-or-command-id", } + if not isinstance(diagnostic_context, dict) or diagnostic_context.get("targetBusinessTraceId") in (None, ""): + return { + "queried": False, + "reason": "missing-unambiguous-target-business-trace-id", + } namespace, namespace_source = guess_agentrun_namespace(identity, provider_decision) run_q = urllib.parse.quote(str(run_id), safe="") command_q = urllib.parse.quote(str(command_id), safe="") command_response, command_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/commands/%s" % (run_q, command_q), timeout=8) - result_response, result_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/commands/%s/result" % (run_q, command_q), timeout=12) - events_response, events_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/events?afterSeq=0&limit=1000" % run_q, timeout=12) + result_response, result_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/commands/%s/result" % (run_q, command_q), timeout=12, diagnostic_context=diagnostic_context) + run_result_response, run_result_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/result?commandId=%s" % (run_q, command_q), timeout=12, diagnostic_context=diagnostic_context) + events_response, events_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/events?afterSeq=0&limit=1000" % run_q, timeout=12, diagnostic_context=diagnostic_context) command_payload = unwrap_payload(command_response.get("json")) result_payload = unwrap_payload(result_response.get("json")) events = event_items(events_response.get("json")) @@ -1227,6 +1268,11 @@ def agentrun_authority_summary(identity, provider_decision): terminal_event.get("status") if isinstance(terminal_event, dict) else None, command_state if command_state in ("completed", "failed", "blocked", "cancelled", "canceled") else None, ) + diagnostic_transport_failures = diagnostic_transport_failure_names([ + ("command-result", result_response), + ("run-result", run_result_response), + ("events", events_response), + ]) return { "queried": True, "namespace": namespace, @@ -1235,7 +1281,10 @@ def agentrun_authority_summary(identity, provider_decision): "ok": bool(command_response.get("ok") or result_response.get("ok") or events_response.get("ok")), "commandOk": command_response.get("ok"), "resultOk": result_response.get("ok"), + "runResultOk": run_result_response.get("ok"), "eventsOk": events_response.get("ok"), + "diagnosticTransportFailed": len(diagnostic_transport_failures) > 0, + "diagnosticTransportFailures": diagnostic_transport_failures, "commandState": command_state, "terminalStatus": terminal_status, "resultTerminalStatus": result_status, @@ -1252,9 +1301,10 @@ def agentrun_authority_summary(identity, provider_decision): "eventCount": len(events), "scopedEventCount": len(scoped_events), "attempts": { - "command": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in command_attempts], - "result": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in result_attempts], - "events": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in events_attempts], + "command": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "diagnosticContextApplied": item.get("diagnosticContextApplied"), "fallbackDenied": item.get("fallbackDenied"), "error": item.get("error"), "stderrTail": item.get("stderrTail")} for item in command_attempts], + "result": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "diagnosticContextApplied": item.get("diagnosticContextApplied"), "fallbackDenied": item.get("fallbackDenied"), "error": item.get("error"), "stderrTail": item.get("stderrTail")} for item in result_attempts], + "runResult": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "diagnosticContextApplied": item.get("diagnosticContextApplied"), "fallbackDenied": item.get("fallbackDenied"), "error": item.get("error"), "stderrTail": item.get("stderrTail")} for item in run_result_attempts], + "events": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "diagnosticContextApplied": item.get("diagnosticContextApplied"), "fallbackDenied": item.get("fallbackDenied"), "error": item.get("error"), "stderrTail": item.get("stderrTail")} for item in events_attempts], }, } @@ -1556,6 +1606,43 @@ def span_business_trace_id(item): return value.strip() return None +def authority_target_business_trace_id(spans, identity): + if isinstance(BUSINESS_TRACE_ID, str) and BUSINESS_TRACE_ID.startswith("trc_"): + return BUSINESS_TRACE_ID + command_id = identity.get("commandId") if isinstance(identity, dict) else None + command_trace_ids = sorted(set( + trace_id + for item in spans + for trace_id in [span_business_trace_id(item)] + if trace_id is not None and command_id not in (None, "") and span_identity_value(item, "commandId") == str(command_id) + )) + if len(command_trace_ids) == 1: + return command_trace_ids[0] + scoped_trace_ids = sorted(set( + trace_id + for item in spans + for trace_id in [span_business_trace_id(item)] + if trace_id is not None + )) + return scoped_trace_ids[0] if len(scoped_trace_ids) == 1 else None + +def new_diagnostic_context(target_business_trace_id, target_command_id, target_otel_trace_id): + trace_id = secrets.token_hex(16) + while trace_id == "0" * 32 or trace_id == target_otel_trace_id: + trace_id = secrets.token_hex(16) + parent_span_id = secrets.token_hex(8) + while parent_span_id == "0" * 16: + parent_span_id = secrets.token_hex(8) + return { + "traceId": trace_id, + "parentSpanId": parent_span_id, + "traceparent": "00-%s-%s-01" % (trace_id, parent_span_id), + "operation": DIAGNOSTIC_OPERATION, + "targetBusinessTraceId": target_business_trace_id, + "targetCommandId": target_command_id, + "expectedTargetOtelTraceId": target_otel_trace_id, + } + def span_identity_value(item, key): attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {} value = attrs.get(key) @@ -2287,7 +2374,44 @@ http_summary = http_status_summary(spans) read_model = hwlab_read_model_summary(spans) identity = identity_from_spans(spans) provider_decision = provider_decision_summary(spans) -agentrun_authority = agentrun_authority_summary(identity, provider_decision) +target_business_trace_id = authority_target_business_trace_id(spans, identity) +diagnostic_context = new_diagnostic_context( + target_business_trace_id, + identity.get("commandId") if isinstance(identity, dict) else None, + resolved_trace_id, +) +diagnostic_context_ready = ( + diagnostic_context.get("targetBusinessTraceId") not in (None, "") + and diagnostic_context.get("targetCommandId") not in (None, "") +) +diagnostic_trace = { + **diagnostic_context, + "contextReady": diagnostic_context_ready, + "headerReadPaths": ["command-result", "run-result", "events"], + "ordinaryReadPaths": ["command-detail"], + "traceCommand": ( + "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --full" + % diagnostic_context.get("traceId") + if diagnostic_context_ready + else None + ), + "targetTraceCommand": ( + "bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --full" + % resolved_trace_id + ), + "linkQueryClue": ( + "inspect diagnostic spans[].links for traceId=%s and attributes " + "diagnostic.target_business_trace_id=%s, commandId=%s" + % ( + resolved_trace_id, + target_business_trace_id, + diagnostic_context.get("targetCommandId"), + ) + if diagnostic_context_ready + else "diagnostic reads skipped because the business trace and command target were not unambiguous" + ), +} +agentrun_authority = agentrun_authority_summary(identity, provider_decision, diagnostic_context) agentrun = agentrun_summary(spans, agentrun_authority) lag = projection_lag_summary(agentrun, read_model) candidates = root_cause_candidates(http_summary, agentrun, read_model, lag, error_spans, idle_warning_spans) @@ -2441,6 +2565,7 @@ payload = { "observabilityGap": observability_gap, "businessTraceIds": business_trace_ids[:20], "businessTraceScope": business_scope, + "diagnosticTrace": diagnostic_trace, "providerDecision": provider_decision, "identity": identity, "agentrun": { diff --git a/scripts/src/platform-infra-observability/render.ts b/scripts/src/platform-infra-observability/render.ts index 7e65300b..8e93ef11 100644 --- a/scripts/src/platform-infra-observability/render.ts +++ b/scripts/src/platform-infra-observability/render.ts @@ -714,6 +714,7 @@ export function renderDiagnoseCodeAgentTable(input: { result: Record; }): RenderedCliResult { const mapping = asPlainRecord(input.result.mapping); + const diagnosticTrace = asPlainRecord(input.result.diagnosticTrace); const identity = asPlainRecord(input.result.identity); const agentrun = asPlainRecord(input.result.agentrun); const agentrunAuthority = asPlainRecord(agentrun?.authority); @@ -774,6 +775,12 @@ export function renderDiagnoseCodeAgentTable(input: { ` backendProfile=${textValue(identity?.backendProfile)} sourceCommit=${shortenMiddle(textValue(identity?.sourceCommit), 20)}`, ` providerProfile=${textValue(providerDecision?.providerProfile)} model=${textValue(providerDecision?.model)} source=${textValue(providerDecision?.providerProfileSource)} adapter=${textValue(providerDecision?.adapter)} runnerNamespace=${textValue(providerDecision?.runnerNamespace)}`, "", + "Diagnostic trace:", + ` traceId=${textValue(diagnosticTrace?.traceId)} parentSpanId=${textValue(diagnosticTrace?.parentSpanId)} operation=${textValue(diagnosticTrace?.operation)} contextReady=${textValue(diagnosticTrace?.contextReady)}`, + ` target businessTraceId=${textValue(diagnosticTrace?.targetBusinessTraceId)} commandId=${textValue(diagnosticTrace?.targetCommandId)} expectedLinkTraceId=${textValue(diagnosticTrace?.expectedTargetOtelTraceId)}`, + ` diagnosticReads=${joinValues(diagnosticTrace?.headerReadPaths, 60)} ordinaryReads=${joinValues(diagnosticTrace?.ordinaryReadPaths, 40)}`, + ` link=${shortenEnd(textValue(diagnosticTrace?.linkQueryClue), 180)}`, + "", "Root causes:", formatTable(["CODE", "CONF", "SUMMARY"], rootRows.length > 0 ? rootRows : [["-", "-", "-"]]), "", @@ -793,6 +800,8 @@ export function renderDiagnoseCodeAgentTable(input: { const next = asPlainRecord(input.result.next); lines.push("", "Next:"); const nextCommands = [ + textValue(diagnosticTrace?.traceCommand), + textValue(diagnosticTrace?.targetTraceCommand), textValue(next?.diagnoseFull), textValue(next?.traceSummary), textValue(next?.traceReads), @@ -801,7 +810,7 @@ export function renderDiagnoseCodeAgentTable(input: { textValue(next?.projection), ].filter((item) => item !== "-"); if (nextCommands.length > 0) { - for (const command of nextCommands.slice(0, 6)) lines.push(` ${command}`); + for (const command of Array.from(new Set(nextCommands)).slice(0, 6)) lines.push(` ${command}`); } else { lines.push(` ${buildDiagnoseCommand(input.target, input.options, true)}`); if (traceId.length > 0 && traceId !== "-") lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${input.target.id} --trace-id ${traceId}`);