fix: 隔离代码代理诊断 trace
This commit is contained in:
Codex
@@ -90,6 +90,7 @@ export function compactDiagnoseCodeAgentResult(value: unknown): Record<string, u
|
||||
observabilityGap: source.observabilityGap ?? null,
|
||||
businessTraceIds: source.businessTraceIds ?? null,
|
||||
businessTraceScope: source.businessTraceScope ?? null,
|
||||
diagnosticTrace: compactDiagnosticTrace(source.diagnosticTrace),
|
||||
providerDecision: source.providerDecision ?? null,
|
||||
identity: compactDiagnoseIdentity(source.identity),
|
||||
agentrun: compactDiagnoseAgentRun(source.agentrun),
|
||||
@@ -118,6 +119,26 @@ export function compactDiagnoseCodeAgentResult(value: unknown): Record<string, u
|
||||
};
|
||||
}
|
||||
|
||||
export function compactDiagnosticTrace(value: unknown): Record<string, unknown> | null {
|
||||
const diagnostic = asPlainRecord(value);
|
||||
if (diagnostic === null) return null;
|
||||
return {
|
||||
traceId: diagnostic.traceId ?? null,
|
||||
parentSpanId: diagnostic.parentSpanId ?? null,
|
||||
traceparent: diagnostic.traceparent ?? null,
|
||||
operation: diagnostic.operation ?? null,
|
||||
targetBusinessTraceId: diagnostic.targetBusinessTraceId ?? null,
|
||||
targetCommandId: diagnostic.targetCommandId ?? null,
|
||||
expectedTargetOtelTraceId: diagnostic.expectedTargetOtelTraceId ?? null,
|
||||
contextReady: diagnostic.contextReady ?? null,
|
||||
headerReadPaths: limitArray(diagnostic.headerReadPaths, 3),
|
||||
ordinaryReadPaths: limitArray(diagnostic.ordinaryReadPaths, 2),
|
||||
traceCommand: diagnostic.traceCommand ?? null,
|
||||
targetTraceCommand: diagnostic.targetTraceCommand ?? null,
|
||||
linkQueryClue: diagnostic.linkQueryClue ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function compactDiagnoseIdentity(value: unknown): Record<string, unknown> | null {
|
||||
return compactRecord(value, [
|
||||
"runId",
|
||||
@@ -169,7 +190,10 @@ export function compactAgentRunAuthority(value: unknown): Record<string, unknown
|
||||
ok: authority.ok ?? null,
|
||||
commandOk: authority.commandOk ?? null,
|
||||
resultOk: authority.resultOk ?? null,
|
||||
runResultOk: authority.runResultOk ?? null,
|
||||
eventsOk: authority.eventsOk ?? null,
|
||||
diagnosticTransportFailed: authority.diagnosticTransportFailed ?? null,
|
||||
diagnosticTransportFailures: limitArray(authority.diagnosticTransportFailures, 3),
|
||||
commandState: authority.commandState ?? null,
|
||||
commandStatus: authority.commandStatus ?? null,
|
||||
resultTerminalStatus: authority.resultTerminalStatus ?? null,
|
||||
@@ -186,26 +210,43 @@ export function compactAgentRunAuthority(value: unknown): Record<string, unknown
|
||||
error: authority.error ?? null,
|
||||
warnings: limitArray(authority.warnings, 3),
|
||||
};
|
||||
const ok = authority.ok === true && authority.commandOk === true && authority.resultOk === true && authority.eventsOk === true;
|
||||
const ok = authority.ok === true
|
||||
&& authority.commandOk === true
|
||||
&& authority.resultOk === true
|
||||
&& authority.runResultOk === true
|
||||
&& authority.eventsOk === true;
|
||||
if (!ok) {
|
||||
result.attempts = compactAgentRunAuthorityAttempts(authority.attempts);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function compactAgentRunAuthorityAttempts(value: unknown): unknown[] {
|
||||
return asArray(value).slice(0, 3).map((item) => {
|
||||
const attempt = asPlainRecord(item) ?? {};
|
||||
return {
|
||||
name: attempt.name ?? attempt.section ?? attempt.kind ?? null,
|
||||
ok: attempt.ok ?? null,
|
||||
status: attempt.status ?? null,
|
||||
exitCode: attempt.exitCode ?? null,
|
||||
url: attempt.url ?? null,
|
||||
error: shortenEnd(textValue(attempt.error), 240),
|
||||
stderrTail: shortenEnd(textValue(attempt.stderrTail ?? attempt.stderr), 500),
|
||||
};
|
||||
});
|
||||
export function compactAgentRunAuthorityAttempts(value: unknown): unknown {
|
||||
const grouped = asPlainRecord(value);
|
||||
if (grouped !== null) {
|
||||
return Object.fromEntries(Object.entries(grouped).slice(0, 4).map(([name, attempts]) => [
|
||||
name,
|
||||
asArray(attempts).slice(0, 3).map((attempt) => compactAgentRunAuthorityAttempt(attempt)),
|
||||
]));
|
||||
}
|
||||
return asArray(value).slice(0, 3).map((attempt) => compactAgentRunAuthorityAttempt(attempt));
|
||||
}
|
||||
|
||||
function compactAgentRunAuthorityAttempt(value: unknown): Record<string, unknown> {
|
||||
const attempt = asPlainRecord(value) ?? {};
|
||||
return {
|
||||
name: attempt.name ?? attempt.section ?? attempt.kind ?? null,
|
||||
ok: attempt.ok ?? null,
|
||||
status: attempt.status ?? null,
|
||||
exitCode: attempt.exitCode ?? attempt.rc ?? null,
|
||||
method: attempt.method ?? null,
|
||||
url: attempt.url ?? null,
|
||||
proxyPath: attempt.proxyPath ?? null,
|
||||
diagnosticContextApplied: attempt.diagnosticContextApplied ?? null,
|
||||
fallbackDenied: attempt.fallbackDenied ?? null,
|
||||
error: shortenEnd(textValue(attempt.error), 240),
|
||||
stderrTail: shortenEnd(textValue(attempt.stderrTail ?? attempt.stderr), 500),
|
||||
};
|
||||
}
|
||||
|
||||
export function compactDiagnoseCandidate(value: unknown): Record<string, unknown> {
|
||||
|
||||
@@ -27,6 +27,11 @@ describe("diagnose-code-agent warm-session terminal classification", () => {
|
||||
namespaceSource: "provider-decision",
|
||||
providerDecisionNamespace: "agentrun-v02",
|
||||
ok: true,
|
||||
commandOk: true,
|
||||
resultOk: true,
|
||||
runResultOk: true,
|
||||
eventsOk: true,
|
||||
diagnosticTransportFailed: false,
|
||||
terminalStatus: "completed",
|
||||
terminalEventSeq: 10,
|
||||
eventCount: 2,
|
||||
@@ -42,6 +47,7 @@ describe("diagnose-code-agent warm-session terminal classification", () => {
|
||||
});
|
||||
expect(diagnosis.calls).toContain("-n agentrun-v02 exec deploy/agentrun-mgr");
|
||||
expect(diagnosis.calls).not.toContain("agentrun-v01");
|
||||
assertDiagnosticRequestContract(diagnosis);
|
||||
|
||||
const compact = compactDiagnoseCodeAgentResult(diagnosis.payload);
|
||||
expect((compact.agentrun as Record<string, any>).authority).toMatchObject({
|
||||
@@ -57,6 +63,10 @@ describe("diagnose-code-agent warm-session terminal classification", () => {
|
||||
result: compact,
|
||||
});
|
||||
expect(rendered.renderedText).toContain("authorityQuery namespace=agentrun-v02 source=provider-decision ok=true commandState=completed terminal=completed");
|
||||
expect(rendered.renderedText).toContain("Diagnostic trace:");
|
||||
expect(rendered.renderedText).toContain(`traceId=${diagnosis.payload.diagnosticTrace.traceId}`);
|
||||
expect(rendered.renderedText).toContain("expectedLinkTraceId=de11d47d1bab0ca4d3f6259279f3a332");
|
||||
expect(rendered.renderedText).toContain(`--target NC01 --trace-id ${diagnosis.payload.diagnosticTrace.traceId} --full`);
|
||||
});
|
||||
|
||||
test("reports unknown instead of running when command authority and terminal span are both absent", () => {
|
||||
@@ -73,6 +83,20 @@ describe("diagnose-code-agent warm-session terminal classification", () => {
|
||||
terminalSpanSeen: false,
|
||||
});
|
||||
expect(diagnosis.payload.rootCauseCandidates[0].code).toBe("observability_gap_authority_query_failed");
|
||||
expect(diagnosis.payload.agentrun.authority).toMatchObject({
|
||||
diagnosticTransportFailed: true,
|
||||
diagnosticTransportFailures: ["command-result", "run-result", "events"],
|
||||
attempts: {
|
||||
result: [{ diagnosticContextApplied: true, fallbackDenied: "headerless-service-proxy", error: "diagnostic-transport-failed" }],
|
||||
runResult: [{ diagnosticContextApplied: true, fallbackDenied: "headerless-service-proxy", error: "diagnostic-transport-failed" }],
|
||||
events: [{ diagnosticContextApplied: true, fallbackDenied: "headerless-service-proxy", error: "diagnostic-transport-failed" }],
|
||||
},
|
||||
});
|
||||
expect(diagnosis.payload.agentrun.authority.attempts.command[0].diagnosticContextApplied).toBe(false);
|
||||
expect(diagnosis.payload.agentrun.authority.attempts.result).toHaveLength(1);
|
||||
expect(diagnosis.payload.agentrun.authority.attempts.runResult).toHaveLength(1);
|
||||
expect(diagnosis.payload.agentrun.authority.attempts.events).toHaveLength(1);
|
||||
assertNoHeaderlessDiagnosticFallback(diagnosis.calls);
|
||||
});
|
||||
|
||||
test("keeps a scoped OTel terminal span complete even if the supplemental authority read fails", () => {
|
||||
@@ -90,6 +114,15 @@ describe("diagnose-code-agent warm-session terminal classification", () => {
|
||||
terminalSpanCount: 1,
|
||||
});
|
||||
});
|
||||
|
||||
test("uses a fresh diagnostic trace for each repeated diagnosis without unscoped authority reads", () => {
|
||||
const first = runFixture({ authorityAvailable: true, terminalSpan: false });
|
||||
const second = runFixture({ authorityAvailable: true, terminalSpan: false });
|
||||
assertDiagnosticRequestContract(first);
|
||||
assertDiagnosticRequestContract(second);
|
||||
expect(first.payload.diagnosticTrace.traceId).not.toBe(second.payload.diagnosticTrace.traceId);
|
||||
expect(first.payload.diagnosticTrace.traceparent).not.toBe(second.payload.diagnosticTrace.traceparent);
|
||||
});
|
||||
});
|
||||
|
||||
function runFixture(input: { authorityAvailable: boolean; terminalSpan: boolean }): {
|
||||
@@ -102,6 +135,7 @@ function runFixture(input: { authorityAvailable: boolean; terminalSpan: boolean
|
||||
writeFileSync(join(directory, "trace.json"), JSON.stringify(traceFixture(input.terminalSpan)));
|
||||
writeFileSync(join(directory, "command.json"), JSON.stringify({ id: "cmd_second", state: "completed" }));
|
||||
writeFileSync(join(directory, "result.json"), JSON.stringify({ ok: true, commandId: "cmd_second", status: "completed", commandState: "completed", terminalStatus: "completed" }));
|
||||
writeFileSync(join(directory, "run-result.json"), JSON.stringify({ ok: true, commandId: "cmd_second", commandState: "completed", terminalStatus: "completed" }));
|
||||
writeFileSync(join(directory, "events.json"), JSON.stringify({
|
||||
items: [
|
||||
{ seq: 10, type: "terminal_status", payload: { commandId: "cmd_second", terminalStatus: "completed" } },
|
||||
@@ -121,6 +155,7 @@ if [ "$AUTHORITY_AVAILABLE" != "1" ]; then
|
||||
fi
|
||||
case "$*" in
|
||||
*"-n agentrun-v02 exec deploy/agentrun-mgr"*"/commands/cmd_second/result"*) cat "$FIXTURE_DIR/result.json"; exit 0 ;;
|
||||
*"-n agentrun-v02 exec deploy/agentrun-mgr"*"/runs/run_warm/result?commandId=cmd_second"*) cat "$FIXTURE_DIR/run-result.json"; exit 0 ;;
|
||||
*"-n agentrun-v02 exec deploy/agentrun-mgr"*"/commands/cmd_second"*) cat "$FIXTURE_DIR/command.json"; exit 0 ;;
|
||||
*"-n agentrun-v02 exec deploy/agentrun-mgr"*"/runs/run_warm/events"*) cat "$FIXTURE_DIR/events.json"; exit 0 ;;
|
||||
esac
|
||||
@@ -147,6 +182,59 @@ exit 1
|
||||
};
|
||||
}
|
||||
|
||||
function assertDiagnosticRequestContract(diagnosis: { payload: Record<string, any>; calls: string }): void {
|
||||
const diagnostic = diagnosis.payload.diagnosticTrace;
|
||||
expect(diagnostic).toMatchObject({
|
||||
operation: "unidesk.diagnose-code-agent",
|
||||
targetBusinessTraceId: "trc_warm_second",
|
||||
targetCommandId: "cmd_second",
|
||||
expectedTargetOtelTraceId: "de11d47d1bab0ca4d3f6259279f3a332",
|
||||
contextReady: true,
|
||||
headerReadPaths: ["command-result", "run-result", "events"],
|
||||
ordinaryReadPaths: ["command-detail"],
|
||||
});
|
||||
expect(diagnostic.traceId).toMatch(/^[0-9a-f]{32}$/);
|
||||
expect(diagnostic.parentSpanId).toMatch(/^[0-9a-f]{16}$/);
|
||||
expect(diagnostic.traceparent).toBe(`00-${diagnostic.traceId}-${diagnostic.parentSpanId}-01`);
|
||||
expect(diagnostic.traceCommand).toContain(`--target NC01 --trace-id ${diagnostic.traceId} --full`);
|
||||
expect(diagnostic.targetTraceCommand).toContain("--target NC01 --trace-id de11d47d1bab0ca4d3f6259279f3a332 --full");
|
||||
expect(diagnostic.linkQueryClue).toContain("links for traceId=de11d47d1bab0ca4d3f6259279f3a332");
|
||||
|
||||
const lines = callLines(diagnosis.calls);
|
||||
const diagnosticLines = lines.filter((line) => line.includes("x-agentrun-trace-purpose: diagnostic"));
|
||||
expect(diagnosticLines).toHaveLength(3);
|
||||
for (const path of [
|
||||
"/api/v1/runs/run_warm/commands/cmd_second/result",
|
||||
"/api/v1/runs/run_warm/result?commandId=cmd_second",
|
||||
"/api/v1/runs/run_warm/events?afterSeq=0&limit=1000",
|
||||
]) {
|
||||
const line = diagnosticLines.find((candidate) => candidate.includes(path));
|
||||
expect(line).toBeDefined();
|
||||
expect(line).toContain(`traceparent: ${diagnostic.traceparent}`);
|
||||
expect(line).toContain("x-agentrun-diagnostic-operation: unidesk.diagnose-code-agent");
|
||||
expect(line).toContain("x-agentrun-diagnostic-target-business-trace-id: trc_warm_second");
|
||||
expect(line).toContain("x-agentrun-diagnostic-target-command-id: cmd_second");
|
||||
}
|
||||
const traceparents = new Set(diagnosticLines.flatMap((line) => line.match(/00-[0-9a-f]{32}-[0-9a-f]{16}-01/g) ?? []));
|
||||
expect([...traceparents]).toEqual([diagnostic.traceparent]);
|
||||
|
||||
const commandDetailLines = lines.filter((line) => line.includes("/api/v1/runs/run_warm/commands/cmd_second") && !line.includes("/commands/cmd_second/result"));
|
||||
expect(commandDetailLines).toHaveLength(1);
|
||||
expect(commandDetailLines[0]).not.toContain("x-agentrun-trace-purpose");
|
||||
assertNoHeaderlessDiagnosticFallback(diagnosis.calls);
|
||||
}
|
||||
|
||||
function assertNoHeaderlessDiagnosticFallback(calls: string): void {
|
||||
const proxyLines = callLines(calls).filter((line) => line.includes("services/http:agentrun-mgr"));
|
||||
expect(proxyLines.some((line) => line.includes("/commands/cmd_second/result"))).toBe(false);
|
||||
expect(proxyLines.some((line) => line.includes("/runs/run_warm/result?commandId=cmd_second"))).toBe(false);
|
||||
expect(proxyLines.some((line) => line.includes("/runs/run_warm/events?afterSeq=0&limit=1000"))).toBe(false);
|
||||
}
|
||||
|
||||
function callLines(calls: string): string[] {
|
||||
return calls.split("\n").map((line) => line.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
function traceFixture(terminalSpan: boolean): Record<string, unknown> {
|
||||
const common = [
|
||||
attribute("traceId", "trc_warm_second"),
|
||||
|
||||
@@ -911,7 +911,7 @@ export function diagnoseCodeAgentScript(observability: ObservabilityConfig, targ
|
||||
return `
|
||||
set -u
|
||||
python3 - <<'PY'
|
||||
import collections, json, re, subprocess, time, urllib.parse
|
||||
import collections, json, re, secrets, subprocess, time, urllib.parse
|
||||
|
||||
BUSINESS_TRACE_ID = ${businessTraceIdLiteral}
|
||||
TRACE_ID = ${traceIdLiteral}
|
||||
@@ -932,6 +932,7 @@ FULL = ${options.full ? "True" : "False"}
|
||||
RAW = ${options.raw ? "True" : "False"}
|
||||
LIMIT = ${options.limit}
|
||||
CANDIDATE_LIMIT = ${options.candidateLimit}
|
||||
DIAGNOSTIC_OPERATION = "unidesk.diagnose-code-agent"
|
||||
DEADLINE = time.time() + 50
|
||||
IMPORTANT_ATTRS = [
|
||||
"traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
|
||||
@@ -1077,16 +1078,38 @@ def agentrun_service_proxy_paths(namespace, api_path):
|
||||
"/api/v1/namespaces/%s/services/http:agentrun-mgr:8080/proxy%s" % (namespace, api_path),
|
||||
]
|
||||
|
||||
def run_agentrun_mgr_json(namespace, api_path, timeout=12):
|
||||
def diagnostic_header_items(diagnostic_context):
|
||||
if not isinstance(diagnostic_context, dict):
|
||||
return []
|
||||
return [
|
||||
("traceparent", diagnostic_context.get("traceparent")),
|
||||
("x-agentrun-trace-purpose", "diagnostic"),
|
||||
("x-agentrun-diagnostic-operation", diagnostic_context.get("operation")),
|
||||
("x-agentrun-diagnostic-target-business-trace-id", diagnostic_context.get("targetBusinessTraceId")),
|
||||
("x-agentrun-diagnostic-target-command-id", diagnostic_context.get("targetCommandId")),
|
||||
]
|
||||
|
||||
def run_agentrun_mgr_json(namespace, api_path, timeout=12, diagnostic_context=None):
|
||||
url = "http://127.0.0.1:8080%s" % api_path
|
||||
diagnostic_headers = [
|
||||
(name, str(value))
|
||||
for name, value in diagnostic_header_items(diagnostic_context)
|
||||
if value not in (None, "")
|
||||
]
|
||||
script_parts = []
|
||||
script_parts.append("url=" + shell_quote(url) + "; ")
|
||||
script_parts.append("api_key=\\\"\${AGENTRUN_API_KEY:-}\\\"; ")
|
||||
script_parts.append("if [ -z \\\"$api_key\\\" ]; then api_key=\\\"\${HWLAB_API_KEY:-}\\\"; fi; ")
|
||||
script_parts.append("if command -v wget >/dev/null 2>&1; then ")
|
||||
script_parts.append("wget -qO- --header=\\\"Authorization: Bearer $api_key\\\" \\\"$url\\\"; ")
|
||||
script_parts.append("wget -qO- --header=\\\"Authorization: Bearer $api_key\\\" ")
|
||||
for name, value in diagnostic_headers:
|
||||
script_parts.append("--header=" + shell_quote(name + ": " + value) + " ")
|
||||
script_parts.append("\\\"$url\\\"; ")
|
||||
script_parts.append("else ")
|
||||
script_parts.append("curl -fsS -H \\\"Authorization: Bearer $api_key\\\" \\\"$url\\\"; ")
|
||||
script_parts.append("curl -fsS -H \\\"Authorization: Bearer $api_key\\\" ")
|
||||
for name, value in diagnostic_headers:
|
||||
script_parts.append("-H " + shell_quote(name + ": " + value) + " ")
|
||||
script_parts.append("\\\"$url\\\"; ")
|
||||
script_parts.append("fi")
|
||||
script = "".join(script_parts)
|
||||
try:
|
||||
@@ -1104,18 +1127,23 @@ def run_agentrun_mgr_json(namespace, api_path, timeout=12):
|
||||
"stderrTail": (proc.stderr or "")[-1000:],
|
||||
"bodyBytes": len((proc.stdout or "").encode("utf-8")),
|
||||
"method": "manager-exec-localhost",
|
||||
"diagnosticContextApplied": bool(diagnostic_headers),
|
||||
}
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
return {"ok": False, "rc": 124, "json": None, "stderrTail": "timeout while querying agentrun manager", "bodyBytes": len((exc.stdout or "").encode("utf-8")), "method": "manager-exec-localhost"}
|
||||
return {"ok": False, "rc": 124, "json": None, "stderrTail": "timeout while querying agentrun manager", "bodyBytes": len((exc.stdout or "").encode("utf-8")), "method": "manager-exec-localhost", "diagnosticContextApplied": bool(diagnostic_headers)}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "rc": 125, "json": None, "stderrTail": str(exc), "bodyBytes": 0, "method": "manager-exec-localhost"}
|
||||
return {"ok": False, "rc": 125, "json": None, "stderrTail": str(exc), "bodyBytes": 0, "method": "manager-exec-localhost", "diagnosticContextApplied": bool(diagnostic_headers)}
|
||||
|
||||
def get_agentrun_json(namespace, api_path, timeout=10):
|
||||
def get_agentrun_json(namespace, api_path, timeout=10, diagnostic_context=None):
|
||||
attempts = []
|
||||
exec_response = run_agentrun_mgr_json(namespace, api_path, timeout=timeout)
|
||||
exec_response = run_agentrun_mgr_json(namespace, api_path, timeout=timeout, diagnostic_context=diagnostic_context)
|
||||
attempts.append(exec_response)
|
||||
if exec_response.get("ok"):
|
||||
return exec_response, attempts
|
||||
if isinstance(diagnostic_context, dict):
|
||||
exec_response["error"] = "diagnostic-transport-failed"
|
||||
exec_response["fallbackDenied"] = "headerless-service-proxy"
|
||||
return exec_response, attempts
|
||||
for proxy_path in agentrun_service_proxy_paths(namespace, api_path):
|
||||
response = run_kubectl_json(proxy_path, timeout=timeout)
|
||||
response["proxyPath"] = proxy_path
|
||||
@@ -1196,7 +1224,14 @@ def preferred_failure_kind(values):
|
||||
return kind
|
||||
return kinds[-1]
|
||||
|
||||
def agentrun_authority_summary(identity, provider_decision):
|
||||
def diagnostic_transport_failure_names(responses):
|
||||
return [
|
||||
name
|
||||
for name, response in responses
|
||||
if isinstance(response, dict) and response.get("error") == "diagnostic-transport-failed"
|
||||
]
|
||||
|
||||
def agentrun_authority_summary(identity, provider_decision, diagnostic_context):
|
||||
run_id = identity.get("runId") if isinstance(identity, dict) else None
|
||||
command_id = identity.get("commandId") if isinstance(identity, dict) else None
|
||||
if not run_id or not command_id:
|
||||
@@ -1204,12 +1239,18 @@ def agentrun_authority_summary(identity, provider_decision):
|
||||
"queried": False,
|
||||
"reason": "missing-run-or-command-id",
|
||||
}
|
||||
if not isinstance(diagnostic_context, dict) or diagnostic_context.get("targetBusinessTraceId") in (None, ""):
|
||||
return {
|
||||
"queried": False,
|
||||
"reason": "missing-unambiguous-target-business-trace-id",
|
||||
}
|
||||
namespace, namespace_source = guess_agentrun_namespace(identity, provider_decision)
|
||||
run_q = urllib.parse.quote(str(run_id), safe="")
|
||||
command_q = urllib.parse.quote(str(command_id), safe="")
|
||||
command_response, command_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/commands/%s" % (run_q, command_q), timeout=8)
|
||||
result_response, result_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/commands/%s/result" % (run_q, command_q), timeout=12)
|
||||
events_response, events_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/events?afterSeq=0&limit=1000" % run_q, timeout=12)
|
||||
result_response, result_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/commands/%s/result" % (run_q, command_q), timeout=12, diagnostic_context=diagnostic_context)
|
||||
run_result_response, run_result_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/result?commandId=%s" % (run_q, command_q), timeout=12, diagnostic_context=diagnostic_context)
|
||||
events_response, events_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/events?afterSeq=0&limit=1000" % run_q, timeout=12, diagnostic_context=diagnostic_context)
|
||||
command_payload = unwrap_payload(command_response.get("json"))
|
||||
result_payload = unwrap_payload(result_response.get("json"))
|
||||
events = event_items(events_response.get("json"))
|
||||
@@ -1227,6 +1268,11 @@ def agentrun_authority_summary(identity, provider_decision):
|
||||
terminal_event.get("status") if isinstance(terminal_event, dict) else None,
|
||||
command_state if command_state in ("completed", "failed", "blocked", "cancelled", "canceled") else None,
|
||||
)
|
||||
diagnostic_transport_failures = diagnostic_transport_failure_names([
|
||||
("command-result", result_response),
|
||||
("run-result", run_result_response),
|
||||
("events", events_response),
|
||||
])
|
||||
return {
|
||||
"queried": True,
|
||||
"namespace": namespace,
|
||||
@@ -1235,7 +1281,10 @@ def agentrun_authority_summary(identity, provider_decision):
|
||||
"ok": bool(command_response.get("ok") or result_response.get("ok") or events_response.get("ok")),
|
||||
"commandOk": command_response.get("ok"),
|
||||
"resultOk": result_response.get("ok"),
|
||||
"runResultOk": run_result_response.get("ok"),
|
||||
"eventsOk": events_response.get("ok"),
|
||||
"diagnosticTransportFailed": len(diagnostic_transport_failures) > 0,
|
||||
"diagnosticTransportFailures": diagnostic_transport_failures,
|
||||
"commandState": command_state,
|
||||
"terminalStatus": terminal_status,
|
||||
"resultTerminalStatus": result_status,
|
||||
@@ -1252,9 +1301,10 @@ def agentrun_authority_summary(identity, provider_decision):
|
||||
"eventCount": len(events),
|
||||
"scopedEventCount": len(scoped_events),
|
||||
"attempts": {
|
||||
"command": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in command_attempts],
|
||||
"result": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in result_attempts],
|
||||
"events": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in events_attempts],
|
||||
"command": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "diagnosticContextApplied": item.get("diagnosticContextApplied"), "fallbackDenied": item.get("fallbackDenied"), "error": item.get("error"), "stderrTail": item.get("stderrTail")} for item in command_attempts],
|
||||
"result": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "diagnosticContextApplied": item.get("diagnosticContextApplied"), "fallbackDenied": item.get("fallbackDenied"), "error": item.get("error"), "stderrTail": item.get("stderrTail")} for item in result_attempts],
|
||||
"runResult": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "diagnosticContextApplied": item.get("diagnosticContextApplied"), "fallbackDenied": item.get("fallbackDenied"), "error": item.get("error"), "stderrTail": item.get("stderrTail")} for item in run_result_attempts],
|
||||
"events": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "diagnosticContextApplied": item.get("diagnosticContextApplied"), "fallbackDenied": item.get("fallbackDenied"), "error": item.get("error"), "stderrTail": item.get("stderrTail")} for item in events_attempts],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1556,6 +1606,43 @@ def span_business_trace_id(item):
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
def authority_target_business_trace_id(spans, identity):
|
||||
if isinstance(BUSINESS_TRACE_ID, str) and BUSINESS_TRACE_ID.startswith("trc_"):
|
||||
return BUSINESS_TRACE_ID
|
||||
command_id = identity.get("commandId") if isinstance(identity, dict) else None
|
||||
command_trace_ids = sorted(set(
|
||||
trace_id
|
||||
for item in spans
|
||||
for trace_id in [span_business_trace_id(item)]
|
||||
if trace_id is not None and command_id not in (None, "") and span_identity_value(item, "commandId") == str(command_id)
|
||||
))
|
||||
if len(command_trace_ids) == 1:
|
||||
return command_trace_ids[0]
|
||||
scoped_trace_ids = sorted(set(
|
||||
trace_id
|
||||
for item in spans
|
||||
for trace_id in [span_business_trace_id(item)]
|
||||
if trace_id is not None
|
||||
))
|
||||
return scoped_trace_ids[0] if len(scoped_trace_ids) == 1 else None
|
||||
|
||||
def new_diagnostic_context(target_business_trace_id, target_command_id, target_otel_trace_id):
|
||||
trace_id = secrets.token_hex(16)
|
||||
while trace_id == "0" * 32 or trace_id == target_otel_trace_id:
|
||||
trace_id = secrets.token_hex(16)
|
||||
parent_span_id = secrets.token_hex(8)
|
||||
while parent_span_id == "0" * 16:
|
||||
parent_span_id = secrets.token_hex(8)
|
||||
return {
|
||||
"traceId": trace_id,
|
||||
"parentSpanId": parent_span_id,
|
||||
"traceparent": "00-%s-%s-01" % (trace_id, parent_span_id),
|
||||
"operation": DIAGNOSTIC_OPERATION,
|
||||
"targetBusinessTraceId": target_business_trace_id,
|
||||
"targetCommandId": target_command_id,
|
||||
"expectedTargetOtelTraceId": target_otel_trace_id,
|
||||
}
|
||||
|
||||
def span_identity_value(item, key):
|
||||
attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {}
|
||||
value = attrs.get(key)
|
||||
@@ -2287,7 +2374,44 @@ http_summary = http_status_summary(spans)
|
||||
read_model = hwlab_read_model_summary(spans)
|
||||
identity = identity_from_spans(spans)
|
||||
provider_decision = provider_decision_summary(spans)
|
||||
agentrun_authority = agentrun_authority_summary(identity, provider_decision)
|
||||
target_business_trace_id = authority_target_business_trace_id(spans, identity)
|
||||
diagnostic_context = new_diagnostic_context(
|
||||
target_business_trace_id,
|
||||
identity.get("commandId") if isinstance(identity, dict) else None,
|
||||
resolved_trace_id,
|
||||
)
|
||||
diagnostic_context_ready = (
|
||||
diagnostic_context.get("targetBusinessTraceId") not in (None, "")
|
||||
and diagnostic_context.get("targetCommandId") not in (None, "")
|
||||
)
|
||||
diagnostic_trace = {
|
||||
**diagnostic_context,
|
||||
"contextReady": diagnostic_context_ready,
|
||||
"headerReadPaths": ["command-result", "run-result", "events"],
|
||||
"ordinaryReadPaths": ["command-detail"],
|
||||
"traceCommand": (
|
||||
"bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --full"
|
||||
% diagnostic_context.get("traceId")
|
||||
if diagnostic_context_ready
|
||||
else None
|
||||
),
|
||||
"targetTraceCommand": (
|
||||
"bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id %s --full"
|
||||
% resolved_trace_id
|
||||
),
|
||||
"linkQueryClue": (
|
||||
"inspect diagnostic spans[].links for traceId=%s and attributes "
|
||||
"diagnostic.target_business_trace_id=%s, commandId=%s"
|
||||
% (
|
||||
resolved_trace_id,
|
||||
target_business_trace_id,
|
||||
diagnostic_context.get("targetCommandId"),
|
||||
)
|
||||
if diagnostic_context_ready
|
||||
else "diagnostic reads skipped because the business trace and command target were not unambiguous"
|
||||
),
|
||||
}
|
||||
agentrun_authority = agentrun_authority_summary(identity, provider_decision, diagnostic_context)
|
||||
agentrun = agentrun_summary(spans, agentrun_authority)
|
||||
lag = projection_lag_summary(agentrun, read_model)
|
||||
candidates = root_cause_candidates(http_summary, agentrun, read_model, lag, error_spans, idle_warning_spans)
|
||||
@@ -2441,6 +2565,7 @@ payload = {
|
||||
"observabilityGap": observability_gap,
|
||||
"businessTraceIds": business_trace_ids[:20],
|
||||
"businessTraceScope": business_scope,
|
||||
"diagnosticTrace": diagnostic_trace,
|
||||
"providerDecision": provider_decision,
|
||||
"identity": identity,
|
||||
"agentrun": {
|
||||
|
||||
@@ -714,6 +714,7 @@ export function renderDiagnoseCodeAgentTable(input: {
|
||||
result: Record<string, unknown>;
|
||||
}): RenderedCliResult {
|
||||
const mapping = asPlainRecord(input.result.mapping);
|
||||
const diagnosticTrace = asPlainRecord(input.result.diagnosticTrace);
|
||||
const identity = asPlainRecord(input.result.identity);
|
||||
const agentrun = asPlainRecord(input.result.agentrun);
|
||||
const agentrunAuthority = asPlainRecord(agentrun?.authority);
|
||||
@@ -774,6 +775,12 @@ export function renderDiagnoseCodeAgentTable(input: {
|
||||
` backendProfile=${textValue(identity?.backendProfile)} sourceCommit=${shortenMiddle(textValue(identity?.sourceCommit), 20)}`,
|
||||
` providerProfile=${textValue(providerDecision?.providerProfile)} model=${textValue(providerDecision?.model)} source=${textValue(providerDecision?.providerProfileSource)} adapter=${textValue(providerDecision?.adapter)} runnerNamespace=${textValue(providerDecision?.runnerNamespace)}`,
|
||||
"",
|
||||
"Diagnostic trace:",
|
||||
` traceId=${textValue(diagnosticTrace?.traceId)} parentSpanId=${textValue(diagnosticTrace?.parentSpanId)} operation=${textValue(diagnosticTrace?.operation)} contextReady=${textValue(diagnosticTrace?.contextReady)}`,
|
||||
` target businessTraceId=${textValue(diagnosticTrace?.targetBusinessTraceId)} commandId=${textValue(diagnosticTrace?.targetCommandId)} expectedLinkTraceId=${textValue(diagnosticTrace?.expectedTargetOtelTraceId)}`,
|
||||
` diagnosticReads=${joinValues(diagnosticTrace?.headerReadPaths, 60)} ordinaryReads=${joinValues(diagnosticTrace?.ordinaryReadPaths, 40)}`,
|
||||
` link=${shortenEnd(textValue(diagnosticTrace?.linkQueryClue), 180)}`,
|
||||
"",
|
||||
"Root causes:",
|
||||
formatTable(["CODE", "CONF", "SUMMARY"], rootRows.length > 0 ? rootRows : [["-", "-", "-"]]),
|
||||
"",
|
||||
@@ -793,6 +800,8 @@ export function renderDiagnoseCodeAgentTable(input: {
|
||||
const next = asPlainRecord(input.result.next);
|
||||
lines.push("", "Next:");
|
||||
const nextCommands = [
|
||||
textValue(diagnosticTrace?.traceCommand),
|
||||
textValue(diagnosticTrace?.targetTraceCommand),
|
||||
textValue(next?.diagnoseFull),
|
||||
textValue(next?.traceSummary),
|
||||
textValue(next?.traceReads),
|
||||
@@ -801,7 +810,7 @@ export function renderDiagnoseCodeAgentTable(input: {
|
||||
textValue(next?.projection),
|
||||
].filter((item) => item !== "-");
|
||||
if (nextCommands.length > 0) {
|
||||
for (const command of nextCommands.slice(0, 6)) lines.push(` ${command}`);
|
||||
for (const command of Array.from(new Set(nextCommands)).slice(0, 6)) lines.push(` ${command}`);
|
||||
} else {
|
||||
lines.push(` ${buildDiagnoseCommand(input.target, input.options, true)}`);
|
||||
if (traceId.length > 0 && traceId !== "-") lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${input.target.id} --trace-id ${traceId}`);
|
||||
|
||||
Reference in New Issue
Block a user