From 3f0f577c24962b81c7fab8cac9266789f507e11a Mon Sep 17 00:00:00 2001 From: lyon Date: Mon, 22 Jun 2026 07:18:11 +0800 Subject: [PATCH] fix: enforce codex turn hard timeout --- src/backend/codex-stdio.ts | 40 ++++++++++++++++++- .../cases/50-hwlab-manual-dispatch.ts | 22 ++++++++-- src/selftest/fake-codex-app-server.ts | 16 +++++++- 3 files changed, 72 insertions(+), 6 deletions(-) diff --git a/src/backend/codex-stdio.ts b/src/backend/codex-stdio.ts index 810b3fe..5df00b7 100644 --- a/src/backend/codex-stdio.ts +++ b/src/backend/codex-stdio.ts @@ -460,7 +460,8 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess const suppressedNotifications = createSuppressedNotificationSummary(); let waitingFor = "codex-app-server"; let lastNotificationMethod: string | null = null; - let lastActivityAt = Date.now(); + const turnStartedAt = Date.now(); + let lastActivityAt = turnStartedAt; let lastToolCall: JsonRecord | null = null; let missingTerminalAfterToolReported = false; let threadId: string | undefined = options.threadId; @@ -530,12 +531,44 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess terminalResolve(); }; options.abortSignal?.addEventListener("abort", abortTurn, { once: true }); - const turnIdleTimeoutMs = positiveTimeout(options.timeoutMs); + const turnHardTimeoutMs = positiveTimeout(options.timeoutMs); + const turnIdleTimeoutMs = turnHardTimeoutMs; const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs); const missingTerminalAfterToolTimeoutMs = codexMissingTerminalAfterToolTimeoutMs(env, turnIdleTimeoutMs); + let hardTimeout: NodeJS.Timeout | null = null; let idleTimeout: NodeJS.Timeout | null = null; let idleWarningTimeout: NodeJS.Timeout | null = null; let missingTerminalAfterToolTimeout: NodeJS.Timeout | null = null; + const turnHardTimeoutAttrs = (): JsonRecord => ({ + waitingFor, + elapsedMs: Math.max(0, Date.now() - turnStartedAt), + idleMs: Math.max(0, Date.now() - lastActivityAt), + timeoutMs: turnHardTimeoutMs, + lastNotificationMethod, + threadId: threadId ?? null, + turnId: turnId ?? null, + terminalStatus: terminal?.status ?? null, + retryable: false, + retryAttempt: null, + retryMaxAttempts: 0, + retryExhausted: true, + lastToolCall, + }); + const failTurnHardTimeout = (): void => { + if (terminal) return; + const elapsedMs = Math.max(0, Date.now() - turnStartedAt); + terminal = { status: "failed", failureKind: "backend-timeout", message: `codex stdio turn hard timed out after ${turnHardTimeoutMs}ms total budget` }; + const attrs = { ...turnHardTimeoutAttrs(), elapsedMs, terminalStatus: terminal.status, failureKind: terminal.failureKind }; + emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "turn:hard-timeout", timeoutMs: turnHardTimeoutMs, elapsedMs, idleMs: Math.max(0, Date.now() - lastActivityAt), waitingFor, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, retryable: false, retryAttempt: null, retryMaxAttempts: 0, retryExhausted: true, lastToolCall } }); + emitCodexOtelSpan("codex_stdio.turn_hard_timeout", options, env, attrs, { status: "error", error: terminal.message }); + beginInterruptAndStop("turn hard timeout", "turn:hard-timeout"); + terminalResolve(); + }; + const scheduleTurnHardTimeout = (): void => { + if (hardTimeout) clearTimeout(hardTimeout); + hardTimeout = setTimeout(failTurnHardTimeout, turnHardTimeoutMs); + hardTimeout.unref?.(); + }; const missingTerminalAfterToolAttrs = (): JsonRecord => ({ waitingFor, idleMs: Math.max(0, Date.now() - lastActivityAt), @@ -603,6 +636,8 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess idleTimeout.unref?.(); }; const stopTurnIdleTimeout = (): void => { + if (hardTimeout) clearTimeout(hardTimeout); + hardTimeout = null; if (!idleTimeout) return; clearTimeout(idleTimeout); idleTimeout = null; @@ -610,6 +645,7 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess idleWarningTimeout = null; clearMissingTerminalAfterToolTimeout(); }; + scheduleTurnHardTimeout(); refreshTurnActivity(); const stopNotifications = session.addNotificationHandler((message) => { refreshTurnActivity(); diff --git a/src/selftest/cases/50-hwlab-manual-dispatch.ts b/src/selftest/cases/50-hwlab-manual-dispatch.ts index eada97f..6657920 100644 --- a/src/selftest/cases/50-hwlab-manual-dispatch.ts +++ b/src/selftest/cases/50-hwlab-manual-dispatch.ts @@ -119,8 +119,11 @@ process.exit(1); const materialized = ((resultEnvelope.resourceBundleRef as JsonRecord).materialized as JsonRecord); assert.deepEqual(((materialized.tools as JsonRecord).names), ["apply_patch", "hwpod", "tran", "trans"]); assert.equal(((materialized.tools as JsonRecord).installed), true); - assert.deepEqual((((materialized.tools as JsonRecord).runtimeTools as JsonRecord).names), ["agentrun-git"]); - assert.equal(((((materialized.tools as JsonRecord).runtimeTools as JsonRecord).items as JsonRecord[])[0]?.overridesResourceTool), false); + const runtimeToolNames = (((materialized.tools as JsonRecord).runtimeTools as JsonRecord).names) as string[]; + assert.ok(runtimeToolNames.includes("agentrun-git")); + const runtimeToolItems = (((materialized.tools as JsonRecord).runtimeTools as JsonRecord).items as JsonRecord[]); + const agentrunGitRuntimeTool = runtimeToolItems.find((item) => item.name === "agentrun-git"); + assert.equal(agentrunGitRuntimeTool?.overridesResourceTool, false); assert.deepEqual(((materialized.skillDirs as JsonRecord).names), ["dad-dev", "hwpod-cli", "hwpod-ctl"]); const requiredSkillItems = ((materialized.requiredSkills as JsonRecord).items as JsonRecord[]); assert.deepEqual(((materialized.requiredSkills as JsonRecord).names), ["dad-dev"]); @@ -273,6 +276,19 @@ process.exit(1); assert.equal(idleEnvelope.failureKind, "backend-timeout"); assert.match(String(idleEnvelope.failureMessage ?? idleEnvelope.message ?? ""), /did not emit turn\/completed/u); + const outputWithoutTerminal = await createHwlabRun(client, context, bundle, "hwlab-session-output-without-terminal", "start a tool that keeps printing without terminal", "hwlab-command-output-without-terminal", 500); + const outputWithoutTerminalRunner = runOnce({ managerUrl: server.baseUrl, runId: outputWithoutTerminal.runId, commandId: outputWithoutTerminal.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-output-without-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-output-without-terminal") }, oneShot: true, pollIntervalMs: 50 }); + await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "acknowledged"); + await waitForEvent(client, outputWithoutTerminal.runId, (event) => event.type === "command_output", "command output without terminal"); + await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "failed"); + const outputWithoutTerminalResult = await outputWithoutTerminalRunner as JsonRecord; + assert.equal(outputWithoutTerminalResult.terminalStatus, "failed"); + assert.equal(outputWithoutTerminalResult.failureKind, "backend-timeout"); + const outputWithoutTerminalEnvelope = await client.get(`/api/v1/runs/${outputWithoutTerminal.runId}/commands/${outputWithoutTerminal.commandId}/result`) as JsonRecord; + assert.equal(outputWithoutTerminalEnvelope.terminalStatus, "failed"); + assert.equal(outputWithoutTerminalEnvelope.failureKind, "backend-timeout"); + assert.match(String(outputWithoutTerminalEnvelope.failureMessage ?? outputWithoutTerminalEnvelope.message ?? ""), /hard timed out/u); + const runningCancel = await createHwlabRun(client, context, bundle, "hwlab-session-cancel-running", "cancel running", "hwlab-command-cancel-running", 10_000); const running = runOnce({ managerUrl: server.baseUrl, runId: runningCancel.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "missing-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-running-cancel") }, oneShot: true }); await waitForCommandState(client, runningCancel.runId, runningCancel.commandId, "acknowledged"); @@ -280,7 +296,7 @@ process.exit(1); const runningResult = await running; assert.equal(runningResult.terminalStatus, "cancelled"); - return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-blocker", "same-run-runner-multiturn", "running-steer", "missing-terminal-after-tool-auto-stop", "running-cancel"] }; + return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-blocker", "same-run-runner-multiturn", "running-steer", "missing-terminal-after-tool-auto-stop", "tool-output-hard-timeout", "running-cancel"] }; } finally { await new Promise((resolve) => server.server.close(() => resolve())); } diff --git a/src/selftest/fake-codex-app-server.ts b/src/selftest/fake-codex-app-server.ts index 6749732..00cd9db 100644 --- a/src/selftest/fake-codex-app-server.ts +++ b/src/selftest/fake-codex-app-server.ts @@ -291,6 +291,20 @@ for await (const line of rl) { }, 25); continue; } + if (mode === "tool-output-without-terminal") { + turnCounter += 1; + const turn = { id: `turn_selftest_${turnCounter}`, status: "running" }; + notify("turn/started", { turn }); + notify("item/started", { item: { id: "tool_output_without_terminal", type: "commandExecution", command: "while true; do echo 0; done", status: "running", processId: process.pid } }); + respond(message.id, { turn }); + let ticks = 0; + const timer = setInterval(() => { + ticks += 1; + notify("item/commandExecution/outputDelta", { itemId: "tool_output_without_terminal", delta: `0 ${ticks}\n` }); + }, 25); + activeSteerTurn = { id: turn.id, completed: false, timer }; + continue; + } if (mode === "steer-waits") { turnCounter += 1; const turn = { id: `turn_selftest_${turnCounter}`, status: "running" }; @@ -344,7 +358,7 @@ for await (const line of rl) { continue; } if (message.method === "turn/interrupt") { - if ((mode !== "tool-hangs-before-turn-start-response" && mode !== "steer-waits") || !activeSteerTurn) { + if ((mode !== "tool-hangs-before-turn-start-response" && mode !== "steer-waits" && mode !== "tool-output-without-terminal") || !activeSteerTurn) { respond(message.id, null, { code: -32000, message: "no active fake turn for interrupt" }); continue; }