fix: enforce codex turn hard timeout

This commit is contained in:
lyon
2026-06-22 07:18:11 +08:00
parent 30bce7eb69
commit 3f0f577c24
3 changed files with 72 additions and 6 deletions
+38 -2
View File
@@ -460,7 +460,8 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
const suppressedNotifications = createSuppressedNotificationSummary();
let waitingFor = "codex-app-server";
let lastNotificationMethod: string | null = null;
let lastActivityAt = Date.now();
const turnStartedAt = Date.now();
let lastActivityAt = turnStartedAt;
let lastToolCall: JsonRecord | null = null;
let missingTerminalAfterToolReported = false;
let threadId: string | undefined = options.threadId;
@@ -530,12 +531,44 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
terminalResolve();
};
options.abortSignal?.addEventListener("abort", abortTurn, { once: true });
const turnIdleTimeoutMs = positiveTimeout(options.timeoutMs);
const turnHardTimeoutMs = positiveTimeout(options.timeoutMs);
const turnIdleTimeoutMs = turnHardTimeoutMs;
const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs);
const missingTerminalAfterToolTimeoutMs = codexMissingTerminalAfterToolTimeoutMs(env, turnIdleTimeoutMs);
let hardTimeout: NodeJS.Timeout | null = null;
let idleTimeout: NodeJS.Timeout | null = null;
let idleWarningTimeout: NodeJS.Timeout | null = null;
let missingTerminalAfterToolTimeout: NodeJS.Timeout | null = null;
const turnHardTimeoutAttrs = (): JsonRecord => ({
waitingFor,
elapsedMs: Math.max(0, Date.now() - turnStartedAt),
idleMs: Math.max(0, Date.now() - lastActivityAt),
timeoutMs: turnHardTimeoutMs,
lastNotificationMethod,
threadId: threadId ?? null,
turnId: turnId ?? null,
terminalStatus: terminal?.status ?? null,
retryable: false,
retryAttempt: null,
retryMaxAttempts: 0,
retryExhausted: true,
lastToolCall,
});
const failTurnHardTimeout = (): void => {
if (terminal) return;
const elapsedMs = Math.max(0, Date.now() - turnStartedAt);
terminal = { status: "failed", failureKind: "backend-timeout", message: `codex stdio turn hard timed out after ${turnHardTimeoutMs}ms total budget` };
const attrs = { ...turnHardTimeoutAttrs(), elapsedMs, terminalStatus: terminal.status, failureKind: terminal.failureKind };
emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "turn:hard-timeout", timeoutMs: turnHardTimeoutMs, elapsedMs, idleMs: Math.max(0, Date.now() - lastActivityAt), waitingFor, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, retryable: false, retryAttempt: null, retryMaxAttempts: 0, retryExhausted: true, lastToolCall } });
emitCodexOtelSpan("codex_stdio.turn_hard_timeout", options, env, attrs, { status: "error", error: terminal.message });
beginInterruptAndStop("turn hard timeout", "turn:hard-timeout");
terminalResolve();
};
const scheduleTurnHardTimeout = (): void => {
if (hardTimeout) clearTimeout(hardTimeout);
hardTimeout = setTimeout(failTurnHardTimeout, turnHardTimeoutMs);
hardTimeout.unref?.();
};
const missingTerminalAfterToolAttrs = (): JsonRecord => ({
waitingFor,
idleMs: Math.max(0, Date.now() - lastActivityAt),
@@ -603,6 +636,8 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
idleTimeout.unref?.();
};
const stopTurnIdleTimeout = (): void => {
if (hardTimeout) clearTimeout(hardTimeout);
hardTimeout = null;
if (!idleTimeout) return;
clearTimeout(idleTimeout);
idleTimeout = null;
@@ -610,6 +645,7 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
idleWarningTimeout = null;
clearMissingTerminalAfterToolTimeout();
};
scheduleTurnHardTimeout();
refreshTurnActivity();
const stopNotifications = session.addNotificationHandler((message) => {
refreshTurnActivity();