import { afterEach, describe, expect, test } from "bun:test"; import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; import { compactDiagnoseCodeAgentResult } from "./apply-status-scripts"; import { diagnoseCodeAgentScript } from "./diagnose-code-agent-script"; import { renderDiagnoseCodeAgentTable } from "./render"; import type { DiagnoseCodeAgentOptions, ObservabilityConfig, ObservabilityTarget } from "./types"; const temporaryDirectories: string[] = []; afterEach(() => { for (const directory of temporaryDirectories.splice(0)) rmSync(directory, { force: true, recursive: true }); }); describe("diagnose-code-agent warm-session terminal classification", () => { test("uses the provider-decision lane and scopes run events by commandId", () => { const diagnosis = runFixture({ authorityAvailable: true, terminalSpan: false }); expect(diagnosis.status).toBe(0); expect(diagnosis.payload.agentrun).toMatchObject({ terminalStatus: "completed", terminalSource: "agentrun-authority", runnerProviderClassification: "completed", authority: { namespace: "agentrun-v02", namespaceSource: "provider-decision", providerDecisionNamespace: "agentrun-v02", ok: true, commandOk: true, resultOk: true, runResultOk: true, eventsOk: true, diagnosticTransportFailed: false, terminalStatus: "completed", terminalEventSeq: 10, eventCount: 2, scopedEventCount: 1, }, }); expect(diagnosis.payload.observabilityGap).toMatchObject({ status: "missing-command-terminal-span", complete: false, serviceCoverageComplete: true, terminalSpanSeen: false, authorityQuery: { namespace: "agentrun-v02", namespaceSource: "provider-decision", ok: true }, }); expect(diagnosis.calls).toContain("-n agentrun-v02 exec deploy/agentrun-mgr"); expect(diagnosis.calls).not.toContain("agentrun-v01"); assertDiagnosticRequestContract(diagnosis); const compact = compactDiagnoseCodeAgentResult(diagnosis.payload); expect((compact.agentrun as Record).authority).toMatchObject({ namespace: "agentrun-v02", namespaceSource: "provider-decision", scopedEventCount: 1, }); const rendered = renderDiagnoseCodeAgentTable({ ok: true, target, options, query: {}, result: compact, }); expect(rendered.renderedText).toContain("authorityQuery namespace=agentrun-v02 source=provider-decision ok=true commandState=completed terminal=completed"); expect(rendered.renderedText).toContain("Diagnostic trace:"); expect(rendered.renderedText).toContain(`traceId=${diagnosis.payload.diagnosticTrace.traceId}`); expect(rendered.renderedText).toContain("expectedLinkTraceId=de11d47d1bab0ca4d3f6259279f3a332"); expect(rendered.renderedText).toContain(`--target NC01 --trace-id ${diagnosis.payload.diagnosticTrace.traceId} --full`); }); test("reports unknown instead of running when command authority and terminal span are both absent", () => { const diagnosis = runFixture({ authorityAvailable: false, terminalSpan: false }); expect(diagnosis.status).toBe(0); expect(diagnosis.payload.agentrun).toMatchObject({ terminalStatus: null, runnerProviderClassification: "unknown", authority: { namespace: "agentrun-v02", namespaceSource: "provider-decision", ok: false }, }); expect(diagnosis.payload.observabilityGap).toMatchObject({ status: "authority-query-failed", complete: false, terminalSpanSeen: false, }); expect(diagnosis.payload.rootCauseCandidates[0].code).toBe("observability_gap_authority_query_failed"); expect(diagnosis.payload.agentrun.authority).toMatchObject({ diagnosticTransportFailed: true, diagnosticTransportFailures: ["command-result", "run-result", "events"], attempts: { result: [{ diagnosticContextApplied: true, fallbackDenied: "headerless-service-proxy", error: "diagnostic-transport-failed" }], runResult: [{ diagnosticContextApplied: true, fallbackDenied: "headerless-service-proxy", error: "diagnostic-transport-failed" }], events: [{ diagnosticContextApplied: true, fallbackDenied: "headerless-service-proxy", error: "diagnostic-transport-failed" }], }, }); expect(diagnosis.payload.agentrun.authority.attempts.command[0].diagnosticContextApplied).toBe(false); expect(diagnosis.payload.agentrun.authority.attempts.result).toHaveLength(1); expect(diagnosis.payload.agentrun.authority.attempts.runResult).toHaveLength(1); expect(diagnosis.payload.agentrun.authority.attempts.events).toHaveLength(1); assertNoHeaderlessDiagnosticFallback(diagnosis.calls); }); test("keeps a scoped OTel terminal span complete even if the supplemental authority read fails", () => { const diagnosis = runFixture({ authorityAvailable: false, terminalSpan: true }); expect(diagnosis.status).toBe(0); expect(diagnosis.payload.agentrun).toMatchObject({ terminalStatus: "completed", terminalSource: "otel-span", runnerProviderClassification: "completed", }); expect(diagnosis.payload.observabilityGap).toMatchObject({ status: "complete", complete: true, terminalSpanSeen: true, terminalSpanCount: 1, }); }); test("uses a fresh diagnostic trace for each repeated diagnosis without unscoped authority reads", () => { const first = runFixture({ authorityAvailable: true, terminalSpan: false }); const second = runFixture({ authorityAvailable: true, terminalSpan: false }); assertDiagnosticRequestContract(first); assertDiagnosticRequestContract(second); expect(first.payload.diagnosticTrace.traceId).not.toBe(second.payload.diagnosticTrace.traceId); expect(first.payload.diagnosticTrace.traceparent).not.toBe(second.payload.diagnosticTrace.traceparent); }); }); function runFixture(input: { authorityAvailable: boolean; terminalSpan: boolean }): { status: number | null; payload: Record; calls: string; } { const directory = mkdtempSync(join(tmpdir(), "unidesk-otel-diagnose-")); temporaryDirectories.push(directory); writeFileSync(join(directory, "trace.json"), JSON.stringify(traceFixture(input.terminalSpan))); writeFileSync(join(directory, "command.json"), JSON.stringify({ id: "cmd_second", state: "completed" })); writeFileSync(join(directory, "result.json"), JSON.stringify({ ok: true, commandId: "cmd_second", status: "completed", commandState: "completed", terminalStatus: "completed" })); writeFileSync(join(directory, "run-result.json"), JSON.stringify({ ok: true, commandId: "cmd_second", commandState: "completed", terminalStatus: "completed" })); writeFileSync(join(directory, "events.json"), JSON.stringify({ items: [ { seq: 10, type: "terminal_status", payload: { commandId: "cmd_second", terminalStatus: "completed" } }, { seq: 11, type: "terminal_status", payload: { commandId: "cmd_first", terminalStatus: "failed", failureKind: "backend-timeout" } }, ], })); const kubectl = join(directory, "kubectl"); writeFileSync(kubectl, `#!/bin/sh set -u echo "$*" >>"$FIXTURE_DIR/calls.log" case "$*" in *"/api/traces/"*) cat "$FIXTURE_DIR/trace.json"; exit 0 ;; esac if [ "$AUTHORITY_AVAILABLE" != "1" ]; then echo "fixture authority unavailable" >&2 exit 1 fi case "$*" in *"-n agentrun-v02 exec deploy/agentrun-mgr"*"/commands/cmd_second/result"*) cat "$FIXTURE_DIR/result.json"; exit 0 ;; *"-n agentrun-v02 exec deploy/agentrun-mgr"*"/runs/run_warm/result?commandId=cmd_second"*) cat "$FIXTURE_DIR/run-result.json"; exit 0 ;; *"-n agentrun-v02 exec deploy/agentrun-mgr"*"/commands/cmd_second"*) cat "$FIXTURE_DIR/command.json"; exit 0 ;; *"-n agentrun-v02 exec deploy/agentrun-mgr"*"/runs/run_warm/events"*) cat "$FIXTURE_DIR/events.json"; exit 0 ;; esac echo "unexpected kubectl call: $*" >&2 exit 1 `); chmodSync(kubectl, 0o755); const script = diagnoseCodeAgentScript(observability, target, null, options); const result = spawnSync("bash", ["-c", script], { encoding: "utf8", env: { ...process.env, AUTHORITY_AVAILABLE: input.authorityAvailable ? "1" : "0", FIXTURE_DIR: directory, PATH: `${directory}:${process.env.PATH ?? ""}`, }, maxBuffer: 4 * 1024 * 1024, }); expect(result.stdout).not.toBe(""); return { status: result.status, payload: JSON.parse(result.stdout), calls: readFileSync(join(directory, "calls.log"), "utf8"), }; } function assertDiagnosticRequestContract(diagnosis: { payload: Record; calls: string }): void { const diagnostic = diagnosis.payload.diagnosticTrace; expect(diagnostic).toMatchObject({ operation: "unidesk.diagnose-code-agent", targetBusinessTraceId: "trc_warm_second", targetCommandId: "cmd_second", expectedTargetOtelTraceId: "de11d47d1bab0ca4d3f6259279f3a332", contextReady: true, headerReadPaths: ["command-result", "run-result", "events"], ordinaryReadPaths: ["command-detail"], }); expect(diagnostic.traceId).toMatch(/^[0-9a-f]{32}$/); expect(diagnostic.parentSpanId).toMatch(/^[0-9a-f]{16}$/); expect(diagnostic.traceparent).toBe(`00-${diagnostic.traceId}-${diagnostic.parentSpanId}-01`); expect(diagnostic.traceCommand).toContain(`--target NC01 --trace-id ${diagnostic.traceId} --full`); expect(diagnostic.targetTraceCommand).toContain("--target NC01 --trace-id de11d47d1bab0ca4d3f6259279f3a332 --full"); expect(diagnostic.linkQueryClue).toContain("links for traceId=de11d47d1bab0ca4d3f6259279f3a332"); const lines = callLines(diagnosis.calls); const diagnosticLines = lines.filter((line) => line.includes("x-agentrun-trace-purpose: diagnostic")); expect(diagnosticLines).toHaveLength(3); for (const path of [ "/api/v1/runs/run_warm/commands/cmd_second/result", "/api/v1/runs/run_warm/result?commandId=cmd_second", "/api/v1/runs/run_warm/events?afterSeq=0&limit=1000", ]) { const line = diagnosticLines.find((candidate) => candidate.includes(path)); expect(line).toBeDefined(); expect(line).toContain(`traceparent: ${diagnostic.traceparent}`); expect(line).toContain("x-agentrun-diagnostic-operation: unidesk.diagnose-code-agent"); expect(line).toContain("x-agentrun-diagnostic-target-business-trace-id: trc_warm_second"); expect(line).toContain("x-agentrun-diagnostic-target-command-id: cmd_second"); } const traceparents = new Set(diagnosticLines.flatMap((line) => line.match(/00-[0-9a-f]{32}-[0-9a-f]{16}-01/g) ?? [])); expect([...traceparents]).toEqual([diagnostic.traceparent]); const commandDetailLines = lines.filter((line) => line.includes("/api/v1/runs/run_warm/commands/cmd_second") && !line.includes("/commands/cmd_second/result")); expect(commandDetailLines).toHaveLength(1); expect(commandDetailLines[0]).not.toContain("x-agentrun-trace-purpose"); assertNoHeaderlessDiagnosticFallback(diagnosis.calls); } function assertNoHeaderlessDiagnosticFallback(calls: string): void { const proxyLines = callLines(calls).filter((line) => line.includes("services/http:agentrun-mgr")); expect(proxyLines.some((line) => line.includes("/commands/cmd_second/result"))).toBe(false); expect(proxyLines.some((line) => line.includes("/runs/run_warm/result?commandId=cmd_second"))).toBe(false); expect(proxyLines.some((line) => line.includes("/runs/run_warm/events?afterSeq=0&limit=1000"))).toBe(false); } function callLines(calls: string): string[] { return calls.split("\n").map((line) => line.trim()).filter(Boolean); } function traceFixture(terminalSpan: boolean): Record { const common = [ attribute("traceId", "trc_warm_second"), attribute("runId", "run_warm"), attribute("commandId", "cmd_second"), attribute("sessionId", "ses_warm"), ]; return { batches: [ batch("hwlab-cloud-api", [span("provider_decision", [ ...common, attribute("agent.chat.provider_profile", "gpt.pika"), attribute("agentRunRunnerNamespace", "agentrun-v02"), ])]), batch("agentrun-manager", [ span("command_created", common), ...(terminalSpan ? [span("runner_terminal.completed", [...common, attribute("terminalStatus", "completed"), attribute("eventType", "terminal_status")])] : []), ]), batch("agentrun-runner", [span("codex_stdio.notification", common)]), ], }; } function batch(service: string, spans: Array>): Record { return { resource: { attributes: [attribute("service.name", service)] }, scopeSpans: [{ scope: { name: `${service}.fixture` }, spans }], }; } function span(name: string, attributes: Array>): Record { return { name, startTimeUnixNano: "1000000", endTimeUnixNano: "2000000", attributes }; } function attribute(key: string, value: string): Record { return { key, value: { stringValue: value } }; } const target: ObservabilityTarget = { id: "NC01", route: "NC01:k3s", namespace: "devops-infra", role: "active", enabled: true, createNamespace: false, }; const options: DiagnoseCodeAgentOptions = { targetId: "NC01", businessTraceId: "trc_warm_second", traceId: "de11d47d1bab0ca4d3f6259279f3a332", runId: null, commandId: null, sessionId: null, runnerJobId: null, full: false, raw: false, limit: 40, candidateLimit: 20, lookbackMinutes: 60, }; const observability = { traceBackend: { serviceName: "tempo" }, probes: { traceQueryPathTemplate: "/api/traces/{{traceId}}" }, } as ObservabilityConfig;