diff --git a/src/backend/codex-stdio.ts b/src/backend/codex-stdio.ts index 28255a5..8e9a2a6 100644 --- a/src/backend/codex-stdio.ts +++ b/src/backend/codex-stdio.ts @@ -471,7 +471,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess const turnStartedAt = Date.now(); let lastActivityAt = turnStartedAt; let lastToolCall: JsonRecord | null = null; - let missingTerminalAfterToolReported = false; let threadId: string | undefined = options.threadId; let turnId: string | undefined; let terminal: { status: TerminalStatus; failureKind: FailureKind | null; message: string | null } | null = null; @@ -567,11 +566,9 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess const turnHardTimeoutMs = positiveTimeout(options.timeoutMs); const turnIdleTimeoutMs = turnHardTimeoutMs; const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs); - const missingTerminalAfterToolTimeoutMs = codexMissingTerminalAfterToolTimeoutMs(env, turnIdleTimeoutMs); let hardTimeout: NodeJS.Timeout | null = null; let idleTimeout: NodeJS.Timeout | null = null; let idleWarningTimeout: NodeJS.Timeout | null = null; - let missingTerminalAfterToolTimeout: NodeJS.Timeout | null = null; let lastToolCallAt: number | null = null; const turnHardTimeoutAttrs = (): JsonRecord => ({ waitingFor, @@ -604,46 +601,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess hardTimeout = setTimeout(failTurnHardTimeout, turnHardTimeoutMs); hardTimeout.unref?.(); }; - const missingTerminalAfterToolAttrs = (): JsonRecord => ({ - waitingFor, - idleMs: Math.max(0, Date.now() - lastActivityAt), - timeoutMs: missingTerminalAfterToolTimeoutMs, - lastNotificationMethod, - threadId: threadId ?? null, - turnId: turnId ?? null, - terminalStatus: terminal?.status ?? null, - retryable: false, - retryAttempt: null, - retryMaxAttempts: 0, - retryExhausted: true, - lastToolCall, - }); - const reportMissingTerminalAfterTool = (): void => { - if (!lastToolCall || missingTerminalAfterToolReported) return; - missingTerminalAfterToolReported = true; - emitCodexOtelSpan("codex_stdio.missing_terminal_after_tool", options, env, missingTerminalAfterToolAttrs()); - }; - const clearMissingTerminalAfterToolTimeout = (): void => { - if (!missingTerminalAfterToolTimeout) return; - clearTimeout(missingTerminalAfterToolTimeout); - missingTerminalAfterToolTimeout = null; - }; - const failMissingTerminalAfterTool = (): void => { - if (terminal || !lastToolCall) return; - reportMissingTerminalAfterTool(); - terminal = { status: "failed", failureKind: "backend-timeout", message: `codex app-server did not emit turn/completed within ${missingTerminalAfterToolTimeoutMs}ms after tool activity` }; - const attrs = { ...missingTerminalAfterToolAttrs(), terminalStatus: terminal.status, failureKind: terminal.failureKind }; - emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "turn:missing-terminal-after-tool-timeout", timeoutMs: missingTerminalAfterToolTimeoutMs, retryable: false, retryAttempt: null, retryMaxAttempts: 0, retryExhausted: true, lastToolCall } }); - emitCodexOtelSpan("codex_stdio.missing_terminal_after_tool_timeout", options, env, attrs, { status: "error", error: terminal.message }); - beginInterruptAndStop("missing terminal after tool timeout", "turn:missing-terminal-after-tool-timeout"); - resolveTerminalNow(); - }; - const scheduleMissingTerminalAfterToolTimeout = (): void => { - if (!lastToolCall) return; - clearMissingTerminalAfterToolTimeout(); - missingTerminalAfterToolTimeout = setTimeout(failMissingTerminalAfterTool, missingTerminalAfterToolTimeoutMs); - missingTerminalAfterToolTimeout.unref?.(); - }; const scheduleIdleWarning = (): void => { if (idleWarningTimeout) clearTimeout(idleWarningTimeout); idleWarningTimeout = setTimeout(() => { @@ -651,7 +608,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess const idleMs = Math.max(0, Date.now() - lastActivityAt); const attrs = { waitingFor, idleMs, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, terminalStatus: null }; emitCodexOtelSpan("codex_stdio.idle_warning", options, env, attrs); - reportMissingTerminalAfterTool(); }, idleWarningMs); idleWarningTimeout.unref?.(); }; @@ -678,7 +634,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess idleTimeout = null; if (idleWarningTimeout) clearTimeout(idleWarningTimeout); idleWarningTimeout = null; - clearMissingTerminalAfterToolTimeout(); }; scheduleTurnHardTimeout(); refreshTurnActivity(); @@ -694,8 +649,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess if (toolSummary?.status === "completed" || toolSummary?.status === "failed") { lastToolCall = toolSummary; lastToolCallAt = Date.now(); - missingTerminalAfterToolReported = false; - scheduleMissingTerminalAfterToolTimeout(); } exposeActiveTurn(normalized.turnId ? "turn-notification" : "notification"); emitEvents(normalized.events); @@ -813,7 +766,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess } } if (!terminal) { - reportMissingTerminalAfterTool(); terminal = { status: "failed", failureKind: "backend-response-invalid", message: "codex app-server did not emit turn/completed" }; } } catch (error) { @@ -1542,12 +1494,6 @@ function codexIdleWarningMs(env: NodeJS.ProcessEnv, turnTimeoutMs: number): numb return Math.max(250, Math.floor(turnTimeoutMs / 2)); } -function codexMissingTerminalAfterToolTimeoutMs(env: NodeJS.ProcessEnv, turnTimeoutMs: number): number { - const configured = Number(env.AGENTRUN_CODEX_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS); - if (Number.isFinite(configured) && configured > 0) return Math.max(250, Math.floor(configured)); - return positiveTimeout(turnTimeoutMs); -} - function codexTerminalNotificationDrainMs(env: NodeJS.ProcessEnv): number { const configured = Number(env.AGENTRUN_CODEX_TERMINAL_NOTIFICATION_DRAIN_MS); if (Number.isFinite(configured) && configured >= 0) return Math.floor(configured); diff --git a/src/mgr/kubernetes-runner-job.ts b/src/mgr/kubernetes-runner-job.ts index cec7c02..d4f68c3 100644 --- a/src/mgr/kubernetes-runner-job.ts +++ b/src/mgr/kubernetes-runner-job.ts @@ -49,7 +49,6 @@ export interface RunnerJobDefaults { jobNamePrefix?: string; lane?: string; runnerIdleTimeoutMs?: number; - missingTerminalAfterToolTimeoutMs?: number; backendRetryMaxAttempts?: number; backendRetryInitialBackoffMs?: number; backendRetryMaxBackoffMs?: number; @@ -68,7 +67,6 @@ export interface CreateRunnerJobInput extends JsonRecord { sourceCommit?: string; serviceAccountName?: string; runnerIdleTimeoutMs?: number; - missingTerminalAfterToolTimeoutMs?: number; backendRetryMaxAttempts?: number; backendRetryInitialBackoffMs?: number; backendRetryMaxBackoffMs?: number; @@ -103,7 +101,6 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore; const attemptId = optionalString(options.input.attemptId) ?? `attempt_${Date.now().toString(36)}`; const runnerId = optionalString(options.input.runnerId); const runnerIdleTimeoutMs = optionalPositiveInteger(options.input.runnerIdleTimeoutMs, "runnerIdleTimeoutMs") ?? options.defaults.runnerIdleTimeoutMs; - const missingTerminalAfterToolTimeoutMs = optionalPositiveInteger(options.input.missingTerminalAfterToolTimeoutMs, "missingTerminalAfterToolTimeoutMs") ?? options.defaults.missingTerminalAfterToolTimeoutMs; const backendRetryMaxAttempts = optionalPositiveInteger(options.input.backendRetryMaxAttempts, "backendRetryMaxAttempts") ?? options.defaults.backendRetryMaxAttempts; const backendRetryInitialBackoffMs = optionalPositiveInteger(options.input.backendRetryInitialBackoffMs, "backendRetryInitialBackoffMs") ?? options.defaults.backendRetryInitialBackoffMs; const backendRetryMaxBackoffMs = optionalPositiveInteger(options.input.backendRetryMaxBackoffMs, "backendRetryMaxBackoffMs") ?? options.defaults.backendRetryMaxBackoffMs; @@ -120,7 +117,6 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore; attemptId: optionalString(options.input.attemptId) ?? null, runnerId: optionalString(options.input.runnerId) ?? null, runnerIdleTimeoutMs: runnerIdleTimeoutMs ?? null, - missingTerminalAfterToolTimeoutMs: missingTerminalAfterToolTimeoutMs ?? null, backendRetryMaxAttempts: backendRetryMaxAttempts ?? null, backendRetryInitialBackoffMs: backendRetryInitialBackoffMs ?? null, backendRetryMaxBackoffMs: backendRetryMaxBackoffMs ?? null, @@ -205,7 +201,6 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore; sourceCommit, transientEnv: renderTransientEnv, ...(runnerIdleTimeoutMs !== undefined ? { runnerIdleTimeoutMs } : {}), - ...(missingTerminalAfterToolTimeoutMs !== undefined ? { missingTerminalAfterToolTimeoutMs } : {}), ...(backendRetryMaxAttempts !== undefined ? { backendRetryMaxAttempts } : {}), ...(backendRetryInitialBackoffMs !== undefined ? { backendRetryInitialBackoffMs } : {}), ...(backendRetryMaxBackoffMs !== undefined ? { backendRetryMaxBackoffMs } : {}), diff --git a/src/mgr/server.ts b/src/mgr/server.ts index a2c4115..377f4f5 100644 --- a/src/mgr/server.ts +++ b/src/mgr/server.ts @@ -56,7 +56,6 @@ function runnerJobDefaultsForRequest(defaults: ManagerServerOptions["runnerJobDe jobNamePrefix, lane, ...(defaults?.runnerIdleTimeoutMs !== undefined ? { runnerIdleTimeoutMs: defaults.runnerIdleTimeoutMs } : optionalPositiveIntegerRecord("runnerIdleTimeoutMs", process.env.AGENTRUN_RUNNER_IDLE_TIMEOUT_MS)), - ...(defaults?.missingTerminalAfterToolTimeoutMs !== undefined ? { missingTerminalAfterToolTimeoutMs: defaults.missingTerminalAfterToolTimeoutMs } : optionalPositiveIntegerRecord("missingTerminalAfterToolTimeoutMs", process.env.AGENTRUN_RUNNER_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS)), ...(defaults?.backendRetryMaxAttempts !== undefined ? { backendRetryMaxAttempts: defaults.backendRetryMaxAttempts } : optionalPositiveIntegerRecord("backendRetryMaxAttempts", process.env.AGENTRUN_BACKEND_RETRY_MAX_ATTEMPTS)), ...(defaults?.backendRetryInitialBackoffMs !== undefined ? { backendRetryInitialBackoffMs: defaults.backendRetryInitialBackoffMs } : optionalPositiveIntegerRecord("backendRetryInitialBackoffMs", process.env.AGENTRUN_BACKEND_RETRY_INITIAL_BACKOFF_MS)), ...(defaults?.backendRetryMaxBackoffMs !== undefined ? { backendRetryMaxBackoffMs: defaults.backendRetryMaxBackoffMs } : optionalPositiveIntegerRecord("backendRetryMaxBackoffMs", process.env.AGENTRUN_BACKEND_RETRY_MAX_BACKOFF_MS)), @@ -114,7 +113,6 @@ export interface ManagerServerOptions { jobNamePrefix?: string; lane?: string; runnerIdleTimeoutMs?: number; - missingTerminalAfterToolTimeoutMs?: number; backendRetryMaxAttempts?: number; backendRetryInitialBackoffMs?: number; backendRetryMaxBackoffMs?: number; diff --git a/src/runner/k8s-job.ts b/src/runner/k8s-job.ts index 6863e28..2107a50 100644 --- a/src/runner/k8s-job.ts +++ b/src/runner/k8s-job.ts @@ -58,7 +58,6 @@ export interface RunnerJobRenderOptions { backoffLimit?: number; ttlSecondsAfterFinished?: number; runnerIdleTimeoutMs?: number; - missingTerminalAfterToolTimeoutMs?: number; backendRetryMaxAttempts?: number; backendRetryInitialBackoffMs?: number; backendRetryMaxBackoffMs?: number; @@ -156,7 +155,7 @@ export function renderRunnerJobDryRun(options: RunnerJobRenderOptions): JsonReco }; } -export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { manifest: JsonRecord; namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; serviceAccountName: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; warnings: string[]; ttlSecondsAfterFinished: number; ttlPolicy: JsonRecord; runnerIdleTimeoutMs: number; missingTerminalAfterToolTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number } { +export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { manifest: JsonRecord; namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; serviceAccountName: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; warnings: string[]; ttlSecondsAfterFinished: number; ttlPolicy: JsonRecord; runnerIdleTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number } { const namespace = options.namespace ?? "agentrun-v01"; const attemptId = options.attemptId ?? `attempt_${Date.now().toString(36)}`; const runnerId = options.runnerId ?? `runner_${shortHash(`${options.run.id}:${attemptId}:${options.commandId}`)}`; @@ -169,7 +168,6 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani const ttlSecondsAfterFinished = normalizeTtlSecondsAfterFinished(options.ttlSecondsAfterFinished, warnings); const ttlPolicy = terminalArtifactTtlPolicy(ttlSecondsAfterFinished); const runnerIdleTimeoutMs = normalizeRunnerIdleTimeoutMs(options.runnerIdleTimeoutMs); - const missingTerminalAfterToolTimeoutMs = normalizeMissingTerminalAfterToolTimeoutMs(options.missingTerminalAfterToolTimeoutMs, runnerIdleTimeoutMs); const backendRetryMaxAttempts = normalizeBackendRetryMaxAttempts(options.backendRetryMaxAttempts); const backendRetryInitialBackoffMs = normalizeBackendRetryBackoffMs(options.backendRetryInitialBackoffMs, "backendRetryInitialBackoffMs", 1000); const backendRetryMaxBackoffMs = normalizeBackendRetryBackoffMs(options.backendRetryMaxBackoffMs, "backendRetryMaxBackoffMs", 30000); @@ -178,7 +176,7 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani const toolCredentials = toolCredentialProjections(options.run, namespace); const sessionPvc = options.sessionPvc; if (secretRefs.length === 0) warnings.push("run executionPolicy.secretScope 未声明 provider SecretRef；runner 将按 secret-unavailable 上报，而不会降级直连外部凭据"); - const env = runnerEnv(options, { namespace, jobName, runnerJobId, runnerId, attemptId, sourceCommit, secretRefs, toolCredentials, sessionPvc, runnerIdleTimeoutMs, missingTerminalAfterToolTimeoutMs, backendRetryMaxAttempts, backendRetryInitialBackoffMs, backendRetryMaxBackoffMs }); + const env = runnerEnv(options, { namespace, jobName, runnerJobId, runnerId, attemptId, sourceCommit, secretRefs, toolCredentials, sessionPvc, runnerIdleTimeoutMs, backendRetryMaxAttempts, backendRetryInitialBackoffMs, backendRetryMaxBackoffMs }); const manifest: JsonRecord = { apiVersion: "batch/v1", kind: "Job", @@ -245,10 +243,10 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani }, }, }; - return { manifest, namespace, jobName, runnerJobId, runnerId, attemptId, sourceCommit, serviceAccountName, secretRefs, toolCredentials, warnings, ttlSecondsAfterFinished, ttlPolicy, runnerIdleTimeoutMs, missingTerminalAfterToolTimeoutMs, backendRetryMaxAttempts, backendRetryInitialBackoffMs, backendRetryMaxBackoffMs }; + return { manifest, namespace, jobName, runnerJobId, runnerId, attemptId, sourceCommit, serviceAccountName, secretRefs, toolCredentials, warnings, ttlSecondsAfterFinished, ttlPolicy, runnerIdleTimeoutMs, backendRetryMaxAttempts, backendRetryInitialBackoffMs, backendRetryMaxBackoffMs }; } -function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; sessionPvc: RunnerSessionPvcOptions | undefined; runnerIdleTimeoutMs: number; missingTerminalAfterToolTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number }): JsonRecord[] { +function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; sessionPvc: RunnerSessionPvcOptions | undefined; runnerIdleTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number }): JsonRecord[] { const selectedSecret = context.secretRefs.find((item) => item.profile === options.run.backendProfile); const codexHome = runnerCodexHome(options.run.backendProfile, selectedSecret, context.sessionPvc); const bootRepoUrl = optionalString(options.bootRepoUrl) ?? defaultBootRepoUrl; @@ -278,7 +276,6 @@ function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string { name: "AGENTRUN_WORK_READY_VERSION", value: String(staticWorkReadyCapabilitySummary().version) }, { name: "AGENTRUN_PROJECT_DEPENDENCY_POLICY", value: "explicit-cache-or-derived-image-only" }, { name: "AGENTRUN_RUNNER_IDLE_TIMEOUT_MS", value: String(context.runnerIdleTimeoutMs) }, - { name: "AGENTRUN_CODEX_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS", value: String(context.missingTerminalAfterToolTimeoutMs) }, { name: "AGENTRUN_BACKEND_RETRY_MAX_ATTEMPTS", value: String(context.backendRetryMaxAttempts) }, { name: "AGENTRUN_BACKEND_RETRY_INITIAL_BACKOFF_MS", value: String(context.backendRetryInitialBackoffMs) }, { name: "AGENTRUN_BACKEND_RETRY_MAX_BACKOFF_MS", value: String(context.backendRetryMaxBackoffMs) }, @@ -309,12 +306,6 @@ function normalizeRunnerIdleTimeoutMs(value: number | undefined): number { return value; } -function normalizeMissingTerminalAfterToolTimeoutMs(value: number | undefined, runnerIdleTimeoutMs: number): number { - if (value === undefined) return runnerIdleTimeoutMs; - if (!Number.isInteger(value) || value <= 0) throw new Error("missingTerminalAfterToolTimeoutMs must be a positive integer"); - return value; -} - function normalizeBackendRetryMaxAttempts(value: number | undefined): number { if (value === undefined) return 1; if (!Number.isInteger(value) || value <= 0) throw new Error("backendRetryMaxAttempts must be a positive integer"); diff --git a/src/selftest/cases/50-hwlab-manual-dispatch.ts b/src/selftest/cases/50-hwlab-manual-dispatch.ts index 6657920..0dbed2e 100644 --- a/src/selftest/cases/50-hwlab-manual-dispatch.ts +++ b/src/selftest/cases/50-hwlab-manual-dispatch.ts @@ -263,18 +263,18 @@ process.exit(1); assert.ok(steerEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "steer-command-acknowledged" && event.payload?.commandId === steerCommand.id && event.payload?.targetCommandId === steerRun.commandId)); assert.ok(steerEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "turn/steer:completed" && event.payload?.commandId === steerCommand.id && event.payload?.targetCommandId === steerRun.commandId && event.payload.deliveryState === "forwarded-to-backend" && event.payload.targetEffect === "not-guaranteed")); - const idleAfterTool = await createHwlabRun(client, context, bundle, "hwlab-session-idle-after-tool", "complete a tool and then fail without terminal", "hwlab-command-idle-after-tool", 10_000); - const idleAfterToolRunner = runOnce({ managerUrl: server.baseUrl, runId: idleAfterTool.runId, commandId: idleAfterTool.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-completes-without-terminal", AGENTRUN_CODEX_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS: "300", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-idle-after-tool") }, oneShot: true, pollIntervalMs: 50 }); - await waitForCommandState(client, idleAfterTool.runId, idleAfterTool.commandId, "acknowledged"); - await waitForEvent(client, idleAfterTool.runId, (event) => event.type === "tool_call" && (event.payload as JsonRecord).status === "completed", "tool_call completed without terminal"); - await waitForCommandState(client, idleAfterTool.runId, idleAfterTool.commandId, "failed"); - const idleAfterToolResult = await idleAfterToolRunner as JsonRecord; - assert.equal(idleAfterToolResult.terminalStatus, "failed"); - assert.equal(idleAfterToolResult.failureKind, "backend-timeout"); - const idleEnvelope = await client.get(`/api/v1/runs/${idleAfterTool.runId}/commands/${idleAfterTool.commandId}/result`) as JsonRecord; + const noEventWatchdog = await createHwlabRun(client, context, bundle, "hwlab-session-no-event-watchdog", "complete a tool and then produce no more events", "hwlab-command-no-event-watchdog", 10_000); + const noEventWatchdogRunner = runOnce({ managerUrl: server.baseUrl, runId: noEventWatchdog.runId, commandId: noEventWatchdog.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-completes-without-terminal", AGENTRUN_RUNNER_IDLE_TIMEOUT_MS: "300", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-no-event-watchdog") }, oneShot: true, pollIntervalMs: 50 }); + await waitForCommandState(client, noEventWatchdog.runId, noEventWatchdog.commandId, "acknowledged"); + await waitForEvent(client, noEventWatchdog.runId, (event) => event.type === "tool_call" && (event.payload as JsonRecord).status === "completed", "tool_call completed before no-event watchdog"); + await waitForCommandState(client, noEventWatchdog.runId, noEventWatchdog.commandId, "failed"); + const noEventWatchdogResult = await noEventWatchdogRunner as JsonRecord; + assert.equal(noEventWatchdogResult.terminalStatus, "failed"); + assert.equal(noEventWatchdogResult.failureKind, "backend-timeout"); + const idleEnvelope = await client.get(`/api/v1/runs/${noEventWatchdog.runId}/commands/${noEventWatchdog.commandId}/result`) as JsonRecord; assert.equal(idleEnvelope.terminalStatus, "failed"); assert.equal(idleEnvelope.failureKind, "backend-timeout"); - assert.match(String(idleEnvelope.failureMessage ?? idleEnvelope.message ?? ""), /did not emit turn\/completed/u); + assert.match(String(idleEnvelope.failureMessage ?? idleEnvelope.message ?? ""), /idle timed out/u); const outputWithoutTerminal = await createHwlabRun(client, context, bundle, "hwlab-session-output-without-terminal", "start a tool that keeps printing without terminal", "hwlab-command-output-without-terminal", 500); const outputWithoutTerminalRunner = runOnce({ managerUrl: server.baseUrl, runId: outputWithoutTerminal.runId, commandId: outputWithoutTerminal.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-output-without-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-output-without-terminal") }, oneShot: true, pollIntervalMs: 50 }); @@ -296,7 +296,7 @@ process.exit(1); const runningResult = await running; assert.equal(runningResult.terminalStatus, "cancelled"); - return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-blocker", "same-run-runner-multiturn", "running-steer", "missing-terminal-after-tool-auto-stop", "tool-output-hard-timeout", "running-cancel"] }; + return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-blocker", "same-run-runner-multiturn", "running-steer", "no-event-watchdog-after-tool", "tool-output-hard-timeout", "running-cancel"] }; } finally { await new Promise((resolve) => server.server.close(() => resolve())); }