fix: enforce codex turn hard timeout
This commit is contained in:
@@ -460,7 +460,8 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
const suppressedNotifications = createSuppressedNotificationSummary();
|
||||
let waitingFor = "codex-app-server";
|
||||
let lastNotificationMethod: string | null = null;
|
||||
let lastActivityAt = Date.now();
|
||||
const turnStartedAt = Date.now();
|
||||
let lastActivityAt = turnStartedAt;
|
||||
let lastToolCall: JsonRecord | null = null;
|
||||
let missingTerminalAfterToolReported = false;
|
||||
let threadId: string | undefined = options.threadId;
|
||||
@@ -530,12 +531,44 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
terminalResolve();
|
||||
};
|
||||
options.abortSignal?.addEventListener("abort", abortTurn, { once: true });
|
||||
const turnIdleTimeoutMs = positiveTimeout(options.timeoutMs);
|
||||
const turnHardTimeoutMs = positiveTimeout(options.timeoutMs);
|
||||
const turnIdleTimeoutMs = turnHardTimeoutMs;
|
||||
const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs);
|
||||
const missingTerminalAfterToolTimeoutMs = codexMissingTerminalAfterToolTimeoutMs(env, turnIdleTimeoutMs);
|
||||
let hardTimeout: NodeJS.Timeout | null = null;
|
||||
let idleTimeout: NodeJS.Timeout | null = null;
|
||||
let idleWarningTimeout: NodeJS.Timeout | null = null;
|
||||
let missingTerminalAfterToolTimeout: NodeJS.Timeout | null = null;
|
||||
const turnHardTimeoutAttrs = (): JsonRecord => ({
|
||||
waitingFor,
|
||||
elapsedMs: Math.max(0, Date.now() - turnStartedAt),
|
||||
idleMs: Math.max(0, Date.now() - lastActivityAt),
|
||||
timeoutMs: turnHardTimeoutMs,
|
||||
lastNotificationMethod,
|
||||
threadId: threadId ?? null,
|
||||
turnId: turnId ?? null,
|
||||
terminalStatus: terminal?.status ?? null,
|
||||
retryable: false,
|
||||
retryAttempt: null,
|
||||
retryMaxAttempts: 0,
|
||||
retryExhausted: true,
|
||||
lastToolCall,
|
||||
});
|
||||
const failTurnHardTimeout = (): void => {
|
||||
if (terminal) return;
|
||||
const elapsedMs = Math.max(0, Date.now() - turnStartedAt);
|
||||
terminal = { status: "failed", failureKind: "backend-timeout", message: `codex stdio turn hard timed out after ${turnHardTimeoutMs}ms total budget` };
|
||||
const attrs = { ...turnHardTimeoutAttrs(), elapsedMs, terminalStatus: terminal.status, failureKind: terminal.failureKind };
|
||||
emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "turn:hard-timeout", timeoutMs: turnHardTimeoutMs, elapsedMs, idleMs: Math.max(0, Date.now() - lastActivityAt), waitingFor, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, retryable: false, retryAttempt: null, retryMaxAttempts: 0, retryExhausted: true, lastToolCall } });
|
||||
emitCodexOtelSpan("codex_stdio.turn_hard_timeout", options, env, attrs, { status: "error", error: terminal.message });
|
||||
beginInterruptAndStop("turn hard timeout", "turn:hard-timeout");
|
||||
terminalResolve();
|
||||
};
|
||||
const scheduleTurnHardTimeout = (): void => {
|
||||
if (hardTimeout) clearTimeout(hardTimeout);
|
||||
hardTimeout = setTimeout(failTurnHardTimeout, turnHardTimeoutMs);
|
||||
hardTimeout.unref?.();
|
||||
};
|
||||
const missingTerminalAfterToolAttrs = (): JsonRecord => ({
|
||||
waitingFor,
|
||||
idleMs: Math.max(0, Date.now() - lastActivityAt),
|
||||
@@ -603,6 +636,8 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
idleTimeout.unref?.();
|
||||
};
|
||||
const stopTurnIdleTimeout = (): void => {
|
||||
if (hardTimeout) clearTimeout(hardTimeout);
|
||||
hardTimeout = null;
|
||||
if (!idleTimeout) return;
|
||||
clearTimeout(idleTimeout);
|
||||
idleTimeout = null;
|
||||
@@ -610,6 +645,7 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
||||
idleWarningTimeout = null;
|
||||
clearMissingTerminalAfterToolTimeout();
|
||||
};
|
||||
scheduleTurnHardTimeout();
|
||||
refreshTurnActivity();
|
||||
const stopNotifications = session.addNotificationHandler((message) => {
|
||||
refreshTurnActivity();
|
||||
|
||||
Reference in New Issue
Block a user