fix: 支持长工具调用 interrupt 与硬超时回收
This commit is contained in:
@@ -181,6 +181,9 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
const liveResult = await livePromise;
|
||||
assert.equal(liveResult.terminalStatus, "completed", "slow live tool event turn should complete");
|
||||
|
||||
await runInterruptBeforeTurnStartResponseCase({ client, managerUrl: server.baseUrl, context });
|
||||
await runHardTimeoutDuringToolProgressCase({ client, managerUrl: server.baseUrl, context });
|
||||
|
||||
const noisy = await createRunWithCommand(client, context, "hello noisy reasoning", "selftest-noisy-reasoning-events", 15_000);
|
||||
const noisyResult = await runOnce({ managerUrl: server.baseUrl, runId: noisy.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "noisy-reasoning-events" }, oneShot: true }) as JsonRecord;
|
||||
assert.equal(noisyResult.terminalStatus, "completed", "noisy reasoning turn should complete");
|
||||
@@ -239,7 +242,7 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
await runSessionStorageSubdirCase({ client, managerUrl: server.baseUrl, context });
|
||||
await runSessionStorageNoSecretLeakCase({ client, managerUrl: server.baseUrl, context });
|
||||
|
||||
return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "runner-lease-conflict-recovery", "codex-stdio-fake-turn", "codex-stdio-k8s-sandbox-override", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-dsflash-go-profile-fake-turn", "codex-stdio-dsflash-go-config-metadata", "codex-stdio-minimax-m3-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-minimax-m3-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-final-agent-message-only", "codex-stdio-web-search-progress", "codex-stdio-stale-thread-resume-failed", "codex-stdio-live-tool-events", "codex-stdio-noisy-reasoning-suppression", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-invalid-tool-call", "codex-stdio-provider-compact-unsupported", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-503-retry-event", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-idle-timeout-progress-refresh", "codex-stdio-command-failure-keeps-run-open", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
|
||||
return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "runner-lease-conflict-recovery", "codex-stdio-fake-turn", "codex-stdio-k8s-sandbox-override", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-dsflash-go-profile-fake-turn", "codex-stdio-dsflash-go-config-metadata", "codex-stdio-minimax-m3-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-minimax-m3-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-final-agent-message-only", "codex-stdio-web-search-progress", "codex-stdio-stale-thread-resume-failed", "codex-stdio-live-tool-events", "codex-stdio-interrupt-before-turn-start-response", "codex-stdio-hard-timeout-during-tool-progress", "codex-stdio-noisy-reasoning-suppression", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-invalid-tool-call", "codex-stdio-provider-compact-unsupported", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-503-retry-event", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-idle-timeout-progress-refresh", "codex-stdio-command-failure-keeps-run-open", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
||||
}
|
||||
@@ -281,7 +284,7 @@ async function runLeaseConflictRecoveryCase(options: { client: ManagerClient; ma
|
||||
}
|
||||
|
||||
async function runSlowProgressIdleCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
||||
const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 60);
|
||||
const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 250);
|
||||
const result = await runOnce({
|
||||
managerUrl: options.managerUrl,
|
||||
runId: item.runId,
|
||||
@@ -296,6 +299,55 @@ async function runSlowProgressIdleCase(options: { client: ManagerClient; manager
|
||||
assert.equal(events.items?.some((event) => event.type === "error" && eventPayload(event).failureKind === "backend-timeout"), false, "progressing turns must not fail on total elapsed time");
|
||||
}
|
||||
|
||||
async function runInterruptBeforeTurnStartResponseCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
||||
const item = await createRunWithCommand(options.client, options.context, "interrupt hanging tool before turn/start response", "selftest-interrupt-before-turn-start-response", 1_000);
|
||||
const runPromise = runOnce({
|
||||
managerUrl: options.managerUrl,
|
||||
runId: item.runId,
|
||||
codexCommand: options.context.fakeCodexCommand,
|
||||
codexArgs: options.context.fakeCodexArgs,
|
||||
codexHome: options.context.codexHome,
|
||||
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-hangs-before-turn-start-response" },
|
||||
oneShot: true,
|
||||
}) as Promise<JsonRecord>;
|
||||
await waitForEvent(options.client, item.runId, (event) => event.type === "tool_call" && eventPayload(event).itemId === "tool_hang_before_response", "tool_call before turn/start response");
|
||||
const interrupt = await options.client.post(`/api/v1/runs/${item.runId}/commands`, { type: "interrupt", payload: { reason: "self-test interrupt hanging tool" }, idempotencyKey: "selftest-interrupt-before-response-command" }) as { id: string };
|
||||
const result = await runPromise;
|
||||
assert.equal(result.terminalStatus, "cancelled", "interrupt should cancel the active turn even if turn/start response has not returned");
|
||||
assert.equal(result.failureKind, "cancelled");
|
||||
const interruptCommand = await options.client.get(`/api/v1/runs/${item.runId}/commands/${interrupt.id}`) as { state?: string };
|
||||
assert.equal(interruptCommand.state, "completed", "interrupt control command should be terminal completed after delivery");
|
||||
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
||||
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "active-turn-control-ready"), "turn/started notification should expose active turn control before turn/start response");
|
||||
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "interrupt-command-acknowledged"), "interrupt command acknowledgement should be visible");
|
||||
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/interrupt:completed"), "interrupt delivery result should be visible");
|
||||
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/start:interrupted-before-response"), "turn/start pending request should be bypassed after interrupt terminal");
|
||||
assertNoSecretLeak({ result, events });
|
||||
}
|
||||
|
||||
async function runHardTimeoutDuringToolProgressCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
||||
const item = await createRunWithCommand(options.client, options.context, "hard timeout during tool progress", "selftest-hard-timeout-tool-progress", 120);
|
||||
const result = await runOnce({
|
||||
managerUrl: options.managerUrl,
|
||||
runId: item.runId,
|
||||
codexCommand: options.context.fakeCodexCommand,
|
||||
codexArgs: options.context.fakeCodexArgs,
|
||||
codexHome: options.context.codexHome,
|
||||
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "hard-timeout-tool-progress" },
|
||||
oneShot: true,
|
||||
}) as JsonRecord;
|
||||
assert.equal(result.terminalStatus, "failed", "hard timeout should fail even while the tool keeps producing progress");
|
||||
assert.equal(result.failureKind, "backend-timeout");
|
||||
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
||||
assert.ok(events.items?.some((event) => event.type === "command_output" && String(eventPayload(event).text ?? "").includes("progress")), "progress output should be visible before hard timeout");
|
||||
assert.ok(events.items?.some((event) => event.type === "error" && eventPayload(event).phase === "turn:hard-timeout"), "hard timeout should be recorded as an error event");
|
||||
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn-interrupt-requested"), "timeout should request backend interrupt before process teardown");
|
||||
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/interrupt:completed"), "timeout interrupt result should be visible");
|
||||
const command = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}`) as { state?: string };
|
||||
assert.equal(command.state, "failed", "hard timed out command should be failed");
|
||||
assertNoSecretLeak({ result, events });
|
||||
}
|
||||
|
||||
async function runFailureDoesNotTerminalRunCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
||||
const item = await createRunWithCommand(options.client, options.context, "first command fails", "selftest-command-failure-keeps-run-open", 3_000);
|
||||
const result = await runOnce({
|
||||
|
||||
Reference in New Issue
Block a user