fix: trace codex stdio lifecycle
This commit is contained in:
lyon
+277
-15
@@ -4,10 +4,11 @@ import { accessSync, constants as fsConstants, readdirSync, readFileSync } from
|
||||
import { chmod, copyFile, mkdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import * as readline from "node:readline";
|
||||
import type { BackendEvent, BackendProfile, BackendTurnResult, FailureKind, InitialPromptAssembly, JsonRecord, JsonValue, TerminalStatus } from "../common/types.js";
|
||||
import type { BackendEvent, BackendProfile, BackendTurnResult, CommandRecord, FailureKind, InitialPromptAssembly, JsonRecord, JsonValue, RunRecord, TerminalStatus } from "../common/types.js";
|
||||
import { redactJson, redactText } from "../common/redaction.js";
|
||||
import { backendProfileSpec } from "../common/backend-profiles.js";
|
||||
import { boundedTextSummary, commandOutputPayload } from "../common/output.js";
|
||||
import { emitAgentRunOtelSpan } from "../common/otel-trace.js";
|
||||
|
||||
const codexProtocol = "codex-app-server-jsonrpc-stdio";
|
||||
const defaultCodexArgs = ["app-server", "--listen", "stdio://"];
|
||||
@@ -16,6 +17,7 @@ const stderrEventChars = 4_000;
|
||||
const requestTimeoutCapMs = 30_000;
|
||||
const assistantDeltaProgressMinChars = 500;
|
||||
const assistantDeltaProgressLimitChars = 1_200;
|
||||
const defaultIdleWarningMs = 8_000;
|
||||
|
||||
const childEnvSummaryKeys = [
|
||||
"CODEX_HOME",
|
||||
@@ -53,11 +55,31 @@ export interface CodexStdioTurnOptions {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
codexHome?: string;
|
||||
initialPrompt?: InitialPromptAssembly;
|
||||
otelContext?: CodexStdioOtelContext;
|
||||
abortSignal?: AbortSignal;
|
||||
onEvent?: (event: BackendEvent) => void | Promise<void>;
|
||||
onActiveTurn?: (control: CodexActiveTurnControl) => void | (() => void);
|
||||
}
|
||||
|
||||
export interface CodexStdioOtelContext extends JsonRecord {
|
||||
run: RunRecord;
|
||||
command: CommandRecord;
|
||||
attemptId?: string | null;
|
||||
runnerId?: string | null;
|
||||
runnerJobId?: string | null;
|
||||
jobName?: string | null;
|
||||
podName?: string | null;
|
||||
sourceCommit?: string | null;
|
||||
logPath?: string | null;
|
||||
}
|
||||
|
||||
interface CodexLifecycleOtelContext {
|
||||
env: NodeJS.ProcessEnv;
|
||||
run: RunRecord;
|
||||
command: CommandRecord;
|
||||
attributes: JsonRecord;
|
||||
}
|
||||
|
||||
export interface CodexActiveTurnControl {
|
||||
threadId: string;
|
||||
turnId: string;
|
||||
@@ -331,6 +353,7 @@ export async function runCodexStdioTurn(options: CodexStdioTurnOptions): Promise
|
||||
export class CodexStdioBackendSession {
|
||||
private client: CodexStdioClient | null = null;
|
||||
private clientKey: string | null = null;
|
||||
private lifecycleOtel: CodexLifecycleOtelContext | null = null;
|
||||
|
||||
async runTurn(options: CodexStdioTurnOptions): Promise<BackendTurnResult> {
|
||||
return await runCodexStdioTurnWithSession(options, this);
|
||||
@@ -343,13 +366,17 @@ export class CodexStdioBackendSession {
|
||||
this.clientKey = null;
|
||||
client.stop();
|
||||
const closeInfo = await client.closedPromise;
|
||||
emitCodexOtelSpanFromLifecycle("codex_app_server.exit", this.lifecycleOtel, { status: closeInfo.failureKind ? "error" : "ok", error: closeInfo.message ?? closeInfo.failureKind ?? undefined, attributes: { ...closeEventAttributes(closeInfo), failureKind: closeInfo.failureKind } });
|
||||
this.lifecycleOtel = null;
|
||||
return [{ type: "backend_status", payload: { phase: "codex-app-server-closed", appServerExit: closeEvent(closeInfo) } }];
|
||||
}
|
||||
|
||||
async getClient(options: CodexStdioTurnOptions, env: NodeJS.ProcessEnv, emitEvent: (event: BackendEvent) => void): Promise<CodexStdioClient> {
|
||||
const key = codexClientKey(options, env);
|
||||
this.lifecycleOtel = codexLifecycleOtelContext(options, env);
|
||||
if (this.client && !this.client.isClosed && this.clientKey === key) {
|
||||
emitEvent({ type: "backend_status", payload: { phase: "codex-app-server:reused", ...backendMetadata(options), protocol: codexProtocol } });
|
||||
emitCodexOtelSpan("codex_app_server.reused", options, env, { command: options.command ?? "codex", argsFingerprint: shortHash(JSON.stringify(options.args ?? defaultCodexArgs)) });
|
||||
return this.client;
|
||||
}
|
||||
const closeEvents = await this.close();
|
||||
@@ -364,6 +391,8 @@ export class CodexStdioBackendSession {
|
||||
config: codexConfigSummary(resolveCodexHome(options), options.backendProfile ?? "codex"),
|
||||
},
|
||||
});
|
||||
const appServerStartMs = Date.now();
|
||||
emitCodexOtelSpan("codex_app_server.starting", options, env, { command: options.command ?? "codex", argsFingerprint: shortHash(JSON.stringify(options.args ?? defaultCodexArgs)), cwd: pathSummary(options.cwd), codexHome: pathSummary(resolveCodexHome(options)) }, { startTimeMs: appServerStartMs });
|
||||
const clientOptions: ConstructorParameters<typeof CodexStdioClient>[0] = {
|
||||
cwd: options.cwd,
|
||||
env,
|
||||
@@ -371,14 +400,20 @@ export class CodexStdioBackendSession {
|
||||
};
|
||||
if (options.command) clientOptions.command = options.command;
|
||||
if (options.args) clientOptions.args = options.args;
|
||||
this.client = new CodexStdioClient(clientOptions);
|
||||
this.clientKey = key;
|
||||
const requestTimeoutMs = Math.min(positiveTimeout(options.timeoutMs), requestTimeoutCapMs);
|
||||
const initializeResult = requireResponseRecord(await this.client.request("initialize", { clientInfo: { name: "agentrun", title: "AgentRun", version: "0.1.0" }, capabilities: { experimentalApi: true } }, requestTimeoutMs), "initialize");
|
||||
validateInitializeResponse(initializeResult);
|
||||
this.client.notify("initialized", {});
|
||||
emitEvent({ type: "backend_status", payload: { phase: "initialize:completed", ...backendMetadata(options), protocol: codexProtocol } });
|
||||
return this.client;
|
||||
try {
|
||||
this.client = new CodexStdioClient(clientOptions);
|
||||
this.clientKey = key;
|
||||
const requestTimeoutMs = Math.min(positiveTimeout(options.timeoutMs), requestTimeoutCapMs);
|
||||
const initializeResult = requireResponseRecord(await this.client.request("initialize", { clientInfo: { name: "agentrun", title: "AgentRun", version: "0.1.0" }, capabilities: { experimentalApi: true } }, requestTimeoutMs), "initialize");
|
||||
validateInitializeResponse(initializeResult);
|
||||
this.client.notify("initialized", {});
|
||||
emitEvent({ type: "backend_status", payload: { phase: "initialize:completed", ...backendMetadata(options), protocol: codexProtocol } });
|
||||
emitCodexOtelSpan("codex_app_server.started", options, env, { command: options.command ?? "codex", argsFingerprint: shortHash(JSON.stringify(options.args ?? defaultCodexArgs)) }, { startTimeMs: appServerStartMs, endTimeMs: Date.now() });
|
||||
return this.client;
|
||||
} catch (error) {
|
||||
emitCodexOtelSpan("codex_app_server.exit", options, env, { phase: "app-server-start", command: options.command ?? "codex", failureKind: normalizeFailure(error).failureKind }, { startTimeMs: appServerStartMs, endTimeMs: Date.now(), status: "error", error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private notificationHandlers = new Set<(message: JsonRecord) => void>();
|
||||
@@ -423,6 +458,11 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
const assistantDeltaProgress = createAssistantDeltaProgressState();
|
||||
const completedAssistantMessages: CompletedAssistantMessage[] = [];
|
||||
const suppressedNotifications = createSuppressedNotificationSummary();
|
||||
let waitingFor = "codex-app-server";
|
||||
let lastNotificationMethod: string | null = null;
|
||||
let lastActivityAt = Date.now();
|
||||
let lastToolCall: JsonRecord | null = null;
|
||||
let missingTerminalAfterToolReported = false;
|
||||
let threadId: string | undefined = options.threadId;
|
||||
let turnId: string | undefined;
|
||||
let terminal: { status: TerminalStatus; failureKind: FailureKind | null; message: string | null } | null = null;
|
||||
@@ -491,29 +531,56 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
};
|
||||
options.abortSignal?.addEventListener("abort", abortTurn, { once: true });
|
||||
const turnIdleTimeoutMs = positiveTimeout(options.timeoutMs);
|
||||
const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs);
|
||||
let idleTimeout: NodeJS.Timeout | null = null;
|
||||
let idleWarningTimeout: NodeJS.Timeout | null = null;
|
||||
const scheduleIdleWarning = (): void => {
|
||||
if (idleWarningTimeout) clearTimeout(idleWarningTimeout);
|
||||
idleWarningTimeout = setTimeout(() => {
|
||||
if (terminal) return;
|
||||
const idleMs = Math.max(0, Date.now() - lastActivityAt);
|
||||
const attrs = { waitingFor, idleMs, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, terminalStatus: null };
|
||||
emitCodexOtelSpan("codex_stdio.idle_warning", options, env, attrs);
|
||||
if (lastToolCall && !missingTerminalAfterToolReported) {
|
||||
missingTerminalAfterToolReported = true;
|
||||
emitCodexOtelSpan("codex_stdio.missing_terminal_after_tool", options, env, { ...attrs, lastToolCall });
|
||||
}
|
||||
}, idleWarningMs);
|
||||
idleWarningTimeout.unref?.();
|
||||
};
|
||||
const refreshTurnActivity = (): void => {
|
||||
if (terminal) return;
|
||||
lastActivityAt = Date.now();
|
||||
scheduleIdleWarning();
|
||||
if (idleTimeout) clearTimeout(idleTimeout);
|
||||
idleTimeout = setTimeout(() => {
|
||||
if (terminal) return;
|
||||
terminal = { status: "failed", failureKind: "backend-timeout", message: `codex stdio turn idle timed out after ${turnIdleTimeoutMs}ms without activity` };
|
||||
emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "turn:idle-timeout" } });
|
||||
emitCodexOtelSpan("codex_stdio.idle_timeout", options, env, { waitingFor, idleMs: Math.max(0, Date.now() - lastActivityAt), lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, terminalStatus: terminal.status, failureKind: terminal.failureKind }, { status: "error", error: terminal.message });
|
||||
beginInterruptAndStop("idle timeout", "turn:idle-timeout");
|
||||
terminalResolve();
|
||||
}, turnIdleTimeoutMs);
|
||||
idleTimeout.unref?.();
|
||||
};
|
||||
const stopTurnIdleTimeout = (): void => {
|
||||
if (!idleTimeout) return;
|
||||
clearTimeout(idleTimeout);
|
||||
idleTimeout = null;
|
||||
if (idleWarningTimeout) clearTimeout(idleWarningTimeout);
|
||||
idleWarningTimeout = null;
|
||||
};
|
||||
refreshTurnActivity();
|
||||
const stopNotifications = session.addNotificationHandler((message) => {
|
||||
refreshTurnActivity();
|
||||
lastNotificationMethod = typeof message.method === "string" ? message.method : "unknown";
|
||||
emitCodexNotificationOtel(options, env, message, { threadId: threadId ?? null, turnId: turnId ?? null, waitingFor });
|
||||
const normalized = normalizeCodexNotification(message, suppressedNotifications);
|
||||
if (normalized.threadId) threadId = normalized.threadId;
|
||||
if (normalized.turnId) turnId = normalized.turnId;
|
||||
waitingFor = waitingForAfterNotification(message, normalized.terminal !== undefined);
|
||||
const toolSummary = toolCallSummaryFromNotification(message);
|
||||
if (toolSummary?.status === "completed" || toolSummary?.status === "failed") lastToolCall = toolSummary;
|
||||
exposeActiveTurn(normalized.turnId ? "turn-notification" : "notification");
|
||||
emitEvents(normalized.events);
|
||||
if (normalized.assistantDelta) {
|
||||
@@ -534,9 +601,19 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
client = await session.getClient(options, env, emitEvent);
|
||||
|
||||
const startThread = async (phasePrefix = "thread/start"): Promise<string> => {
|
||||
const response = requireResponseRecord(await client!.request("thread/start", withOptionalModel({ cwd: options.cwd, approvalPolicy: options.approvalPolicy, sandbox: options.sandbox, serviceName: "agentrun" }, options.model), requestTimeoutMs), "thread/start");
|
||||
waitingFor = "thread/start";
|
||||
const startedAt = Date.now();
|
||||
emitCodexOtelSpan("codex_stdio.thread_start.start", options, env, { waitingFor, requestedThreadId: null }, { startTimeMs: startedAt });
|
||||
let response: JsonRecord;
|
||||
try {
|
||||
response = requireResponseRecord(await client!.request("thread/start", withOptionalModel({ cwd: options.cwd, approvalPolicy: options.approvalPolicy, sandbox: options.sandbox, serviceName: "agentrun" }, options.model), requestTimeoutMs), "thread/start");
|
||||
} catch (error) {
|
||||
emitCodexOtelSpan("codex_stdio.thread_start.failed", options, env, { waitingFor, failureKind: normalizeFailure(error).failureKind }, { startTimeMs: startedAt, endTimeMs: Date.now(), status: "error", error });
|
||||
throw error;
|
||||
}
|
||||
const nextThreadId = requireNestedId(response, "thread/start", "thread");
|
||||
emitEvent({ type: "backend_status", payload: { phase: `${phasePrefix}:completed`, threadId: nextThreadId } });
|
||||
emitCodexOtelSpan("codex_stdio.thread_start.completed", options, env, { waitingFor, threadId: nextThreadId }, { startTimeMs: startedAt, endTimeMs: Date.now() });
|
||||
return nextThreadId;
|
||||
};
|
||||
|
||||
@@ -549,12 +626,17 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
emitEvent({ type: "backend_status", payload: { phase: "codex-rollout-storage-mounted", pvcName: sessionPvcName, pvcNamespace: sessionPvcNamespace, mountPath: sessionPvcMountPath, codexRolloutSubdir: codexRolloutSubdirEnv ?? "sessions", valuesPrinted: false } });
|
||||
}
|
||||
if (options.threadId) {
|
||||
waitingFor = "thread/resume";
|
||||
const startedAt = Date.now();
|
||||
emitCodexOtelSpan("codex_stdio.thread_resume.start", options, env, { waitingFor, requestedThreadId: options.threadId }, { startTimeMs: startedAt });
|
||||
try {
|
||||
const threadResponse = requireResponseRecord(await client.request("thread/resume", withOptionalModel({ threadId: options.threadId, cwd: options.cwd, approvalPolicy: options.approvalPolicy, sandbox: options.sandbox }, options.model), requestTimeoutMs), "thread/resume");
|
||||
threadId = requireNestedId(threadResponse, "thread/resume", "thread");
|
||||
emitEvent({ type: "backend_status", payload: { phase: "thread/resume:completed", threadId } });
|
||||
emitCodexOtelSpan("codex_stdio.thread_resume.completed", options, env, { waitingFor, requestedThreadId: options.threadId, threadId }, { startTimeMs: startedAt, endTimeMs: Date.now() });
|
||||
} catch (error) {
|
||||
const failure = normalizeFailure(error);
|
||||
emitCodexOtelSpan("codex_stdio.thread_resume.failed", options, env, { waitingFor, requestedThreadId: options.threadId, failureKind: failure.failureKind }, { startTimeMs: startedAt, endTimeMs: Date.now(), status: "error", error });
|
||||
if (sessionPvcName && isNoRolloutFoundMessage(failure.message)) {
|
||||
throw new CodexStdioFailure(
|
||||
"session-store-evicted",
|
||||
@@ -571,17 +653,34 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
|
||||
const promptInjection = initialPromptInjection(options.initialPrompt, willResumeThread);
|
||||
emitEvent({ type: "backend_status", payload: { phase: "initial-prompt-assembly", initialPromptInjected: promptInjection.injected, reason: promptInjection.reason, initialPrompt: options.initialPrompt?.summary ?? { available: false, valuesPrinted: false }, valuesPrinted: false } });
|
||||
const turnStart = await Promise.race([
|
||||
client.request("turn/start", withOptionalModel({ threadId, input: textInputForUserMessage(options.prompt, promptInjection), cwd: options.cwd, approvalPolicy: options.approvalPolicy }, options.model), requestTimeoutMs).then((response) => ({ kind: "response" as const, response })),
|
||||
terminalPromise.then(() => ({ kind: "terminal" as const })),
|
||||
]);
|
||||
waitingFor = "turn/start";
|
||||
const turnStartStartedAt = Date.now();
|
||||
emitCodexOtelSpan("codex_stdio.turn_start.start", options, env, { waitingFor, threadId: threadId ?? null }, { startTimeMs: turnStartStartedAt });
|
||||
let turnStart: { kind: "response"; response: unknown } | { kind: "terminal" };
|
||||
try {
|
||||
turnStart = await Promise.race([
|
||||
client.request("turn/start", withOptionalModel({ threadId, input: textInputForUserMessage(options.prompt, promptInjection), cwd: options.cwd, approvalPolicy: options.approvalPolicy }, options.model), requestTimeoutMs).then((response) => ({ kind: "response" as const, response })),
|
||||
terminalPromise.then(() => ({ kind: "terminal" as const })),
|
||||
]);
|
||||
} catch (error) {
|
||||
emitCodexOtelSpan("codex_stdio.turn_start.failed", options, env, { waitingFor, threadId: threadId ?? null, failureKind: normalizeFailure(error).failureKind }, { startTimeMs: turnStartStartedAt, endTimeMs: Date.now(), status: "error", error });
|
||||
throw error;
|
||||
}
|
||||
if (turnStart.kind === "response") {
|
||||
const turnResponse = requireResponseRecord(turnStart.response, "turn/start");
|
||||
turnId = requireNestedId(turnResponse, "turn/start", "turn");
|
||||
emitEvent({ type: "backend_status", payload: { phase: "turn/start:completed", turnId } });
|
||||
waitingFor = "turn/completed";
|
||||
emitCodexOtelSpan("codex_stdio.turn_start.completed", options, env, { waitingFor, threadId: threadId ?? null, turnId }, { startTimeMs: turnStartStartedAt, endTimeMs: Date.now() });
|
||||
exposeActiveTurn("turn-start-response");
|
||||
} else {
|
||||
emitEvent({ type: "backend_status", payload: { phase: "turn/start:interrupted-before-response", threadId: threadId ?? null, turnId: turnId ?? null } });
|
||||
const terminalSnapshot = terminal as unknown as { status: TerminalStatus; failureKind: FailureKind | null; message: string | null } | null;
|
||||
if (terminalSnapshot?.status === "completed") {
|
||||
emitCodexOtelSpan("codex_stdio.turn_start.completed", options, env, { waitingFor: "terminal-before-response", threadId: threadId ?? null, turnId: turnId ?? null, terminalStatus: terminalSnapshot.status, responseSource: "terminal-before-response" }, { startTimeMs: turnStartStartedAt, endTimeMs: Date.now() });
|
||||
} else {
|
||||
emitCodexOtelSpan("codex_stdio.turn_start.failed", options, env, { waitingFor, threadId: threadId ?? null, turnId: turnId ?? null, terminalStatus: terminalSnapshot?.status ?? null, failureKind: terminalSnapshot?.failureKind ?? null }, { startTimeMs: turnStartStartedAt, endTimeMs: Date.now(), status: "error", error: terminalSnapshot?.message ?? "turn/start interrupted before response" });
|
||||
}
|
||||
}
|
||||
|
||||
if (!terminal) {
|
||||
@@ -594,7 +693,13 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "transport:closed-before-terminal", appServerExit: closeEvent(race.closeInfo) } });
|
||||
}
|
||||
}
|
||||
if (!terminal) terminal = { status: "failed", failureKind: "backend-response-invalid", message: "codex app-server did not emit turn/completed" };
|
||||
if (!terminal) {
|
||||
if (lastToolCall && !missingTerminalAfterToolReported) {
|
||||
missingTerminalAfterToolReported = true;
|
||||
emitCodexOtelSpan("codex_stdio.missing_terminal_after_tool", options, env, { waitingFor, idleMs: Math.max(0, Date.now() - lastActivityAt), lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, lastToolCall });
|
||||
}
|
||||
terminal = { status: "failed", failureKind: "backend-response-invalid", message: "codex app-server did not emit turn/completed" };
|
||||
}
|
||||
} catch (error) {
|
||||
if (!terminal) {
|
||||
const failure = normalizeFailure(error);
|
||||
@@ -1216,6 +1321,163 @@ function backendMetadata(options: CodexStdioTurnOptions): JsonRecord {
|
||||
};
|
||||
}
|
||||
|
||||
function emitCodexOtelSpan(name: string, options: CodexStdioTurnOptions, env: NodeJS.ProcessEnv, attributes: JsonRecord = {}, span: { startTimeMs?: number; endTimeMs?: number; status?: "ok" | "error"; error?: unknown } = {}): void {
|
||||
const context = options.otelContext;
|
||||
if (!context) return;
|
||||
void emitAgentRunOtelSpan(name, context.run, env, {
|
||||
command: context.command,
|
||||
scopeName: "agentrun.runner",
|
||||
...(span.startTimeMs !== undefined ? { startTimeMs: span.startTimeMs } : {}),
|
||||
...(span.endTimeMs !== undefined ? { endTimeMs: span.endTimeMs } : {}),
|
||||
...(span.status !== undefined ? { status: span.status } : {}),
|
||||
...(span.error !== undefined ? { error: span.error } : {}),
|
||||
attributes: codexOtelAttributes(options, env, attributes),
|
||||
});
|
||||
}
|
||||
|
||||
function emitCodexOtelSpanFromLifecycle(name: string, context: CodexLifecycleOtelContext | null, span: { startTimeMs?: number; endTimeMs?: number; status?: "ok" | "error"; error?: unknown; attributes?: JsonRecord } = {}): void {
|
||||
if (!context) return;
|
||||
void emitAgentRunOtelSpan(name, context.run, context.env, {
|
||||
command: context.command,
|
||||
scopeName: "agentrun.runner",
|
||||
...(span.startTimeMs !== undefined ? { startTimeMs: span.startTimeMs } : {}),
|
||||
...(span.endTimeMs !== undefined ? { endTimeMs: span.endTimeMs } : {}),
|
||||
...(span.status !== undefined ? { status: span.status } : {}),
|
||||
...(span.error !== undefined ? { error: span.error } : {}),
|
||||
attributes: { ...context.attributes, ...(span.attributes ?? {}), valuesPrinted: false },
|
||||
});
|
||||
}
|
||||
|
||||
function codexLifecycleOtelContext(options: CodexStdioTurnOptions, env: NodeJS.ProcessEnv): CodexLifecycleOtelContext | null {
|
||||
if (!options.otelContext) return null;
|
||||
return {
|
||||
env,
|
||||
run: options.otelContext.run,
|
||||
command: options.otelContext.command,
|
||||
attributes: codexOtelAttributes(options, env),
|
||||
};
|
||||
}
|
||||
|
||||
function codexOtelAttributes(options: CodexStdioTurnOptions, env: NodeJS.ProcessEnv, extra: JsonRecord = {}): JsonRecord {
|
||||
const context = options.otelContext;
|
||||
const profile = options.backendProfile ?? context?.run.backendProfile ?? "codex";
|
||||
return {
|
||||
runId: context?.run.id ?? null,
|
||||
commandId: context?.command.id ?? null,
|
||||
runnerJobId: context?.runnerJobId ?? null,
|
||||
runnerId: context?.runnerId ?? null,
|
||||
attemptId: context?.attemptId ?? null,
|
||||
sessionId: context?.run.sessionRef?.sessionId ?? null,
|
||||
threadId: context?.run.sessionRef?.threadId ?? options.threadId ?? null,
|
||||
backendProfile: profile,
|
||||
codexHome: pathSummary(resolveCodexHome(options)),
|
||||
appServerSessionId: shortHash(codexClientKey(options, env)),
|
||||
jobName: context?.jobName ?? null,
|
||||
podName: context?.podName ?? null,
|
||||
sourceCommit: context?.sourceCommit ?? null,
|
||||
logPath: context?.logPath ? pathSummary(context.logPath) : null,
|
||||
...extra,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function closeEventAttributes(closeInfo: CodexStdioCloseInfo): JsonRecord {
|
||||
return {
|
||||
exitCode: closeInfo.code,
|
||||
signal: closeInfo.signal,
|
||||
stderrBytes: closeInfo.stderrBytes,
|
||||
stderrTruncated: closeInfo.stderrTruncated,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function codexIdleWarningMs(env: NodeJS.ProcessEnv, turnTimeoutMs: number): number {
|
||||
const configured = Number(env.AGENTRUN_CODEX_STDIO_IDLE_WARNING_MS ?? env.AGENTRUN_CODEX_IDLE_WARNING_MS);
|
||||
if (Number.isFinite(configured) && configured > 0) return Math.max(250, Math.floor(configured));
|
||||
if (turnTimeoutMs > defaultIdleWarningMs) return defaultIdleWarningMs;
|
||||
return Math.max(250, Math.floor(turnTimeoutMs / 2));
|
||||
}
|
||||
|
||||
function emitCodexNotificationOtel(options: CodexStdioTurnOptions, env: NodeJS.ProcessEnv, message: JsonRecord, state: JsonRecord): void {
|
||||
const attributes = { ...state, ...notificationOtelAttributes(message) };
|
||||
emitCodexOtelSpan("codex_stdio.notification", options, env, attributes);
|
||||
const method = String(attributes.method ?? "unknown");
|
||||
if (method === "item/agentMessage/delta") emitCodexOtelSpan("codex_stdio.assistant_delta", options, env, attributes);
|
||||
const tool = toolCallSummaryFromNotification(message);
|
||||
if (tool) {
|
||||
const status = tool.status === "failed" ? "failed" : tool.status === "started" ? "started" : "completed";
|
||||
emitCodexOtelSpan(`codex_stdio.tool_call.${status}`, options, env, { ...state, ...tool }, { status: status === "failed" ? "error" : "ok", error: status === "failed" ? "tool call failed" : undefined });
|
||||
}
|
||||
if (method === "turn/completed") {
|
||||
const failureKind = typeof attributes.failureKind === "string" ? attributes.failureKind : null;
|
||||
emitCodexOtelSpan("codex_stdio.turn_completed", options, env, attributes, { status: failureKind ? "error" : "ok", error: failureKind ?? undefined });
|
||||
if (failureKind === "provider-stream-disconnected") emitCodexOtelSpan("codex_stdio.provider_stream_disconnected", options, env, attributes, { status: "error", error: failureKind });
|
||||
}
|
||||
if (method === "error") {
|
||||
const failureKind = typeof attributes.failureKind === "string" ? attributes.failureKind : null;
|
||||
if (failureKind === "provider-stream-disconnected") emitCodexOtelSpan("codex_stdio.provider_stream_disconnected", options, env, attributes, { status: "error", error: failureKind });
|
||||
}
|
||||
}
|
||||
|
||||
function notificationOtelAttributes(message: JsonRecord): JsonRecord {
|
||||
const method = typeof message.method === "string" ? message.method : "unknown";
|
||||
const params = asRecordAt(message, "params");
|
||||
const item = asRecordAt(params, "item");
|
||||
const turn = asRecordAt(params, "turn");
|
||||
const error = asRecordAt(params, "error");
|
||||
const itemType = typeof item.type === "string" ? item.type : null;
|
||||
const turnStatus = typeof turn.status === "string" ? turn.status : null;
|
||||
const failureKind = method === "error"
|
||||
? classifyCodexErrorRecord(error, "backend-failed")
|
||||
: method === "turn/completed" && turnStatus !== "completed"
|
||||
? classifyCodexErrorRecord(Object.keys(error).length > 0 ? error : { message: turnStatus ?? "unknown" }, "backend-failed")
|
||||
: null;
|
||||
return {
|
||||
method,
|
||||
itemId: stringAt(item, "id") ?? stringAt(params, "itemId"),
|
||||
itemType,
|
||||
itemStatus: typeof item.status === "string" ? item.status : null,
|
||||
turnStatus,
|
||||
turnId: stringAt(turn, "id"),
|
||||
failureKind,
|
||||
willRetry: typeof params.willRetry === "boolean" ? params.willRetry : null,
|
||||
deltaChars: typeof params.delta === "string" ? params.delta.length : null,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function toolCallSummaryFromNotification(message: JsonRecord): JsonRecord | null {
|
||||
const method = typeof message.method === "string" ? message.method : "";
|
||||
if (method !== "item/started" && method !== "item/completed") return null;
|
||||
const item = asRecordAt(asRecordAt(message, "params"), "item");
|
||||
const itemType = typeof item.type === "string" ? item.type : "unknown";
|
||||
if (!isVisibleCodexToolItemType(itemType)) return null;
|
||||
const command = toolCallCommandSummary(item, itemType, toolCallName(item, itemType));
|
||||
return {
|
||||
method,
|
||||
itemId: stringAt(item, "id"),
|
||||
itemType,
|
||||
toolName: toolCallName(item, itemType),
|
||||
status: toolCallStatus(method, item),
|
||||
exitCode: typeof item.exitCode === "number" ? item.exitCode : null,
|
||||
durationMs: typeof item.durationMs === "number" ? item.durationMs : null,
|
||||
commandFingerprint: command ? shortHash(command) : null,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function waitingForAfterNotification(message: JsonRecord, terminal: boolean): string {
|
||||
if (terminal) return "terminal";
|
||||
const method = typeof message.method === "string" ? message.method : "unknown";
|
||||
const tool = toolCallSummaryFromNotification(message);
|
||||
if (tool?.status === "started") return "tool-call";
|
||||
if (tool?.status === "completed" || tool?.status === "failed") return "turn/completed";
|
||||
if (method === "item/agentMessage/delta") return "assistant-delta";
|
||||
if (method === "turn/started" || method === "turn/start") return "turn/completed";
|
||||
if (method === "error") return asRecordAt(message, "params").willRetry === true ? "provider-retry" : "terminal";
|
||||
return "codex-notification";
|
||||
}
|
||||
|
||||
function envSummary(env: NodeJS.ProcessEnv): JsonRecord {
|
||||
const keyState: Record<string, JsonValue> = {};
|
||||
for (const key of childEnvSummaryKeys) keyState[key] = { present: typeof env[key] === "string" && String(env[key]).length > 0 };
|
||||
|
||||
Reference in New Issue
Block a user