Merge pull request #224 from pikasTech/fix/tool-hard-timeout-122-20260621-2315

修复 codex turn 持续输出时硬超时失效

src/backend/codex-stdio.ts

@@ -460,7 +460,8 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
   const suppressedNotifications = createSuppressedNotificationSummary();
   let waitingFor = "codex-app-server";
   let lastNotificationMethod: string | null = null;
-  let lastActivityAt = Date.now();
+  const turnStartedAt = Date.now();
+  let lastActivityAt = turnStartedAt;
   let lastToolCall: JsonRecord | null = null;
   let missingTerminalAfterToolReported = false;
   let threadId: string | undefined = options.threadId;
@@ -530,12 +531,44 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
     terminalResolve();
   };
   options.abortSignal?.addEventListener("abort", abortTurn, { once: true });
-  const turnIdleTimeoutMs = positiveTimeout(options.timeoutMs);
+  const turnHardTimeoutMs = positiveTimeout(options.timeoutMs);
+  const turnIdleTimeoutMs = turnHardTimeoutMs;
   const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs);
   const missingTerminalAfterToolTimeoutMs = codexMissingTerminalAfterToolTimeoutMs(env, turnIdleTimeoutMs);
+  let hardTimeout: NodeJS.Timeout | null = null;
   let idleTimeout: NodeJS.Timeout | null = null;
   let idleWarningTimeout: NodeJS.Timeout | null = null;
   let missingTerminalAfterToolTimeout: NodeJS.Timeout | null = null;
+  const turnHardTimeoutAttrs = (): JsonRecord => ({
+    waitingFor,
+    elapsedMs: Math.max(0, Date.now() - turnStartedAt),
+    idleMs: Math.max(0, Date.now() - lastActivityAt),
+    timeoutMs: turnHardTimeoutMs,
+    lastNotificationMethod,
+    threadId: threadId ?? null,
+    turnId: turnId ?? null,
+    terminalStatus: terminal?.status ?? null,
+    retryable: false,
+    retryAttempt: null,
+    retryMaxAttempts: 0,
+    retryExhausted: true,
+    lastToolCall,
+  });
+  const failTurnHardTimeout = (): void => {
+    if (terminal) return;
+    const elapsedMs = Math.max(0, Date.now() - turnStartedAt);
+    terminal = { status: "failed", failureKind: "backend-timeout", message: `codex stdio turn hard timed out after ${turnHardTimeoutMs}ms total budget` };
+    const attrs = { ...turnHardTimeoutAttrs(), elapsedMs, terminalStatus: terminal.status, failureKind: terminal.failureKind };
+    emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "turn:hard-timeout", timeoutMs: turnHardTimeoutMs, elapsedMs, idleMs: Math.max(0, Date.now() - lastActivityAt), waitingFor, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, retryable: false, retryAttempt: null, retryMaxAttempts: 0, retryExhausted: true, lastToolCall } });
+    emitCodexOtelSpan("codex_stdio.turn_hard_timeout", options, env, attrs, { status: "error", error: terminal.message });
+    beginInterruptAndStop("turn hard timeout", "turn:hard-timeout");
+    terminalResolve();
+  };
+  const scheduleTurnHardTimeout = (): void => {
+    if (hardTimeout) clearTimeout(hardTimeout);
+    hardTimeout = setTimeout(failTurnHardTimeout, turnHardTimeoutMs);
+    hardTimeout.unref?.();
+  };
   const missingTerminalAfterToolAttrs = (): JsonRecord => ({
     waitingFor,
     idleMs: Math.max(0, Date.now() - lastActivityAt),
@@ -603,6 +636,8 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
     idleTimeout.unref?.();
   };
   const stopTurnIdleTimeout = (): void => {
+    if (hardTimeout) clearTimeout(hardTimeout);
+    hardTimeout = null;
     if (!idleTimeout) return;
     clearTimeout(idleTimeout);
     idleTimeout = null;
@@ -610,6 +645,7 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
     idleWarningTimeout = null;
     clearMissingTerminalAfterToolTimeout();
   };
+  scheduleTurnHardTimeout();
   refreshTurnActivity();
   const stopNotifications = session.addNotificationHandler((message) => {
     refreshTurnActivity();

src/selftest/cases/50-hwlab-manual-dispatch.ts

@@ -119,8 +119,11 @@ process.exit(1);
     const materialized = ((resultEnvelope.resourceBundleRef as JsonRecord).materialized as JsonRecord);
     assert.deepEqual(((materialized.tools as JsonRecord).names), ["apply_patch", "hwpod", "tran", "trans"]);
     assert.equal(((materialized.tools as JsonRecord).installed), true);
-    assert.deepEqual((((materialized.tools as JsonRecord).runtimeTools as JsonRecord).names), ["agentrun-git"]);
+    const runtimeToolNames = (((materialized.tools as JsonRecord).runtimeTools as JsonRecord).names) as string[];
-    assert.equal(((((materialized.tools as JsonRecord).runtimeTools as JsonRecord).items as JsonRecord[])[0]?.overridesResourceTool), false);
+    assert.ok(runtimeToolNames.includes("agentrun-git"));
+    const runtimeToolItems = (((materialized.tools as JsonRecord).runtimeTools as JsonRecord).items as JsonRecord[]);
+    const agentrunGitRuntimeTool = runtimeToolItems.find((item) => item.name === "agentrun-git");
+    assert.equal(agentrunGitRuntimeTool?.overridesResourceTool, false);
     assert.deepEqual(((materialized.skillDirs as JsonRecord).names), ["dad-dev", "hwpod-cli", "hwpod-ctl"]);
     const requiredSkillItems = ((materialized.requiredSkills as JsonRecord).items as JsonRecord[]);
     assert.deepEqual(((materialized.requiredSkills as JsonRecord).names), ["dad-dev"]);
@@ -273,6 +276,19 @@ process.exit(1);
     assert.equal(idleEnvelope.failureKind, "backend-timeout");
     assert.match(String(idleEnvelope.failureMessage ?? idleEnvelope.message ?? ""), /did not emit turn\/completed/u);
 
+    const outputWithoutTerminal = await createHwlabRun(client, context, bundle, "hwlab-session-output-without-terminal", "start a tool that keeps printing without terminal", "hwlab-command-output-without-terminal", 500);
+    const outputWithoutTerminalRunner = runOnce({ managerUrl: server.baseUrl, runId: outputWithoutTerminal.runId, commandId: outputWithoutTerminal.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-output-without-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-output-without-terminal") }, oneShot: true, pollIntervalMs: 50 });
+    await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "acknowledged");
+    await waitForEvent(client, outputWithoutTerminal.runId, (event) => event.type === "command_output", "command output without terminal");
+    await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "failed");
+    const outputWithoutTerminalResult = await outputWithoutTerminalRunner as JsonRecord;
+    assert.equal(outputWithoutTerminalResult.terminalStatus, "failed");
+    assert.equal(outputWithoutTerminalResult.failureKind, "backend-timeout");
+    const outputWithoutTerminalEnvelope = await client.get(`/api/v1/runs/${outputWithoutTerminal.runId}/commands/${outputWithoutTerminal.commandId}/result`) as JsonRecord;
+    assert.equal(outputWithoutTerminalEnvelope.terminalStatus, "failed");
+    assert.equal(outputWithoutTerminalEnvelope.failureKind, "backend-timeout");
+    assert.match(String(outputWithoutTerminalEnvelope.failureMessage ?? outputWithoutTerminalEnvelope.message ?? ""), /hard timed out/u);
+
     const runningCancel = await createHwlabRun(client, context, bundle, "hwlab-session-cancel-running", "cancel running", "hwlab-command-cancel-running", 10_000);
     const running = runOnce({ managerUrl: server.baseUrl, runId: runningCancel.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "missing-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-running-cancel") }, oneShot: true });
     await waitForCommandState(client, runningCancel.runId, runningCancel.commandId, "acknowledged");
@@ -280,7 +296,7 @@ process.exit(1);
     const runningResult = await running;
     assert.equal(runningResult.terminalStatus, "cancelled");
 
-    return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-blocker", "same-run-runner-multiturn", "running-steer", "missing-terminal-after-tool-auto-stop", "running-cancel"] };
+    return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-blocker", "same-run-runner-multiturn", "running-steer", "missing-terminal-after-tool-auto-stop", "tool-output-hard-timeout", "running-cancel"] };
   } finally {
     await new Promise<void>((resolve) => server.server.close(() => resolve()));
   }

src/selftest/fake-codex-app-server.ts

@@ -291,6 +291,20 @@ for await (const line of rl) {
       }, 25);
       continue;
     }
+    if (mode === "tool-output-without-terminal") {
+      turnCounter += 1;
+      const turn = { id: `turn_selftest_${turnCounter}`, status: "running" };
+      notify("turn/started", { turn });
+      notify("item/started", { item: { id: "tool_output_without_terminal", type: "commandExecution", command: "while true; do echo 0; done", status: "running", processId: process.pid } });
+      respond(message.id, { turn });
+      let ticks = 0;
+      const timer = setInterval(() => {
+        ticks += 1;
+        notify("item/commandExecution/outputDelta", { itemId: "tool_output_without_terminal", delta: `0 ${ticks}\n` });
+      }, 25);
+      activeSteerTurn = { id: turn.id, completed: false, timer };
+      continue;
+    }
     if (mode === "steer-waits") {
       turnCounter += 1;
       const turn = { id: `turn_selftest_${turnCounter}`, status: "running" };
@@ -344,7 +358,7 @@ for await (const line of rl) {
     continue;
   }
   if (message.method === "turn/interrupt") {
-    if ((mode !== "tool-hangs-before-turn-start-response" && mode !== "steer-waits") || !activeSteerTurn) {
+    if ((mode !== "tool-hangs-before-turn-start-response" && mode !== "steer-waits" && mode !== "tool-output-without-terminal") || !activeSteerTurn) {
       respond(message.id, null, { code: -32000, message: "no active fake turn for interrupt" });
       continue;
     }